From a1931e4bf77853a13f6da9e28a8761097efe721f Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Thu, 12 Sep 2019 10:43:38 -0400 Subject: [PATCH] Elastic: Change release prefix from stx to mon and decouple The stx release prefix was overlapping with platform-integ-apps containers, impacting cleanup operations. In addition, this update allows for decoupling of the release prefix so that configuration system overrides settings do not have to know about the prefix. Story: 2005733 Task: 36585 Change-Id: Ic6283f630273bf424d8593a6bdba3d289334d18f Signed-off-by: Kevin Smith --- monitor-helm/centos/monitor-helm.spec | 2 + monitor-helm/files/0005-decouple-config.patch | 97 +++++++++++++++++++ .../manifests/monitor_manifest.yaml | 76 +++++++-------- 3 files changed, 135 insertions(+), 40 deletions(-) create mode 100644 monitor-helm/files/0005-decouple-config.patch diff --git a/monitor-helm/centos/monitor-helm.spec b/monitor-helm/centos/monitor-helm.spec index a6b97ad..924a67e 100644 --- a/monitor-helm/centos/monitor-helm.spec +++ b/monitor-helm/centos/monitor-helm.spec @@ -22,6 +22,7 @@ Patch01: 0001-Add-Makefile-for-helm-charts.patch Patch02: 0002-kibana-workaround-checksum-for-configmap.yaml.patch Patch03: 0003-helm-chart-changes-for-stx-monitor.patch Patch04: 0004-ipv6-helm-chart-changes.patch +Patch05: 0005-decouple-config.patch BuildRequires: helm @@ -34,6 +35,7 @@ Monitor Helm charts %patch02 -p1 %patch03 -p1 %patch04 -p1 +%patch05 -p1 %build # initialize helm and build the toolkit diff --git a/monitor-helm/files/0005-decouple-config.patch b/monitor-helm/files/0005-decouple-config.patch new file mode 100644 index 0000000..718e0f7 --- /dev/null +++ b/monitor-helm/files/0005-decouple-config.patch @@ -0,0 +1,97 @@ +From ac9bce2e01870d8623dc7910cd68079d56f917ca Mon Sep 17 00:00:00 2001 +From: Kevin Smith +Date: Wed, 11 Sep 2019 18:07:50 -0400 +Subject: [PATCH 1/1] decouple config + +--- + stable/elasticsearch/templates/configmap.yaml | 8 ++++++++ + stable/elasticsearch/values.yaml | 1 + + stable/filebeat/templates/configmap.yaml | 2 +- + stable/metricbeat/templates/configmap.yaml | 2 +- + stable/metricbeat/values.yaml | 6 ++++-- + 5 files changed, 15 insertions(+), 4 deletions(-) + +diff --git a/stable/elasticsearch/templates/configmap.yaml b/stable/elasticsearch/templates/configmap.yaml +index 977ee9a..6bc2065 100644 +--- a/stable/elasticsearch/templates/configmap.yaml ++++ b/stable/elasticsearch/templates/configmap.yaml +@@ -87,6 +87,14 @@ data: + gateway.recover_after_time: ${RECOVER_AFTER_TIME:5m} + gateway.recover_after_master_nodes: ${RECOVER_AFTER_MASTER_NODES:2} + gateway.recover_after_data_nodes: ${RECOVER_AFTER_DATA_NODES:1} ++ ++{{- if .Values.cluster.auto_config_initial_masters }} ++ cluster.initial_master_nodes: ++ {{- range $i := until (.Values.master.replicas|int) }} ++ - {{template "elasticsearch.master.fullname" $ }}-{{ $i }} ++ {{- end }} ++{{- end }} ++ + {{- with .Values.cluster.config }} + {{ toYaml . | indent 4 }} + {{- end }} +diff --git a/stable/elasticsearch/values.yaml b/stable/elasticsearch/values.yaml +index 6dcd9b0..37321dc 100644 +--- a/stable/elasticsearch/values.yaml ++++ b/stable/elasticsearch/values.yaml +@@ -70,6 +70,7 @@ cluster: + plugins: [] + # - ingest-attachment + # - mapper-size ++ auto_config_initial_masters: true + + client: + name: client +diff --git a/stable/filebeat/templates/configmap.yaml b/stable/filebeat/templates/configmap.yaml +index 9433176..8377e5a 100644 +--- a/stable/filebeat/templates/configmap.yaml ++++ b/stable/filebeat/templates/configmap.yaml +@@ -22,7 +22,7 @@ data: + + # Check if this is a map of entries. If so, we'll only export the template + # to the first address +- if [[ ${ADDR:0:1} == [ ]]; then ++ if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then + # Remove the square brackets + ADDR=${ADDR:1:-1} + # Take the first entry in case there are more than one +diff --git a/stable/metricbeat/templates/configmap.yaml b/stable/metricbeat/templates/configmap.yaml +index f990f0c..4249e7f 100644 +--- a/stable/metricbeat/templates/configmap.yaml ++++ b/stable/metricbeat/templates/configmap.yaml +@@ -22,7 +22,7 @@ data: + + # Check if this is a map of entries. If so, we'll only export the template + # to the first address +- if [[ ${ADDR:0:1} == [ ]]; then ++ if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then + # Remove the square brackets + ADDR=${ADDR:1:-1} + # Take the first entry in case there are more than one +diff --git a/stable/metricbeat/values.yaml b/stable/metricbeat/values.yaml +index 2908a1b..c8f758c 100644 +--- a/stable/metricbeat/values.yaml ++++ b/stable/metricbeat/values.yaml +@@ -113,7 +113,7 @@ deployment: + # Uncomment this to get k8s events: + # - event + period: 10s +- hosts: ["kube-state-metrics:8080"] ++ hosts: ["${KUBE_STATE_METRICS_HOST}:8080"] + # If overrideModules is not empty, metricbeat chart's default modules won't be used at all. + overrideModules: {} + +@@ -122,7 +122,9 @@ plugins: [] + # - kinesis.so + + # additional environment +-# extraEnv: ++extraEnv: ++ - name: KUBE_STATE_METRICS_HOST ++ value: "mon-kube-state-metrics" + # - name: test1 + # value: "test1" + # - name: test2 +-- +1.8.3.1 + diff --git a/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml b/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml index 97f1ff5..0ac82b9 100644 --- a/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml +++ b/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml @@ -10,7 +10,7 @@ data: wait: timeout: 600 labels: - release: stx-nginx-ingress + release: mon-nginx-ingress install: no_hooks: false upgrade: @@ -19,7 +19,7 @@ data: delete: - type: job labels: - release: stx-nginx-ingress + release: mon-nginx-ingress values: controller: kind: DaemonSet @@ -64,7 +64,7 @@ data: wait: timeout: 600 labels: - release: stx-elasticsearch + release: mon-elasticsearch test: enabled: false install: @@ -75,20 +75,16 @@ data: delete: - type: job labels: - release: stx-elasticsearch + release: mon-elasticsearch - type: pod labels: - release: stx-elasticsearch + release: mon-elasticsearch component: test values: image: repository: "docker.elastic.co/elasticsearch/elasticsearch-oss" tag: 7.2.0 cluster: - config: - cluster.initial_master_nodes: - - stx-elasticsearch-master-0 - - stx-elasticsearch-master-1 env: MINIMUM_MASTER_NODES: "1" EXPECTED_MASTER_NODES: "1" @@ -102,9 +98,9 @@ data: nginx.ingress.kubernetes.io/ssl-redirect: "false" nginx.ingress.kubernetes.io/force-ssl-redirect: "false" # Note in order to differentiate from kibana, prefix with - # stx-elasticsearch-client, but strip before presenting + # mon-elasticsearch-client, but strip before presenting # to elasticsearch with above rewrite-target - path: /stx-elasticsearch-client(/|$)(.*) + path: /mon-elasticsearch-client(/|$)(.*) hosts: - "" master: @@ -132,7 +128,7 @@ data: wait: timeout: 600 labels: - release: stx-filebeat + release: mon-filebeat test: enabled: false install: @@ -143,10 +139,10 @@ data: delete: - type: job labels: - release: stx-filebeat + release: mon-filebeat - type: pod labels: - release: stx-filebeat + release: mon-filebeat component: test values: image: @@ -157,14 +153,14 @@ data: enabled: false output.logstash: enabled: true - hosts: ["stx-logstash:5044"] + hosts: ["mon-logstash:5044"] output.elasticsearch: enabled: false - hosts: ["http://stx-elasticsearch-client:9200"] + hosts: ["http://mon-elasticsearch-client:9200"] ilm.pattern: "000001" setup.kibana: # Note port 5601 is default - host: "stx-kibana" + host: "mon-kibana" setup.dashboards: enabled: true filebeat.autodiscover: @@ -181,7 +177,7 @@ data: extraVolumes: - configMap: defaultMode: 493 - name: stx-filebeat + name: mon-filebeat name: setupscript tolerations: - key: "services" @@ -206,7 +202,7 @@ data: wait: timeout: 600 labels: - release: stx-metricbeat + release: mon-metricbeat test: enabled: false install: @@ -217,10 +213,10 @@ data: delete: - type: job labels: - release: stx-metricbeat + release: mon-metricbeat - type: pod labels: - release: stx-metricbeat + release: mon-metricbeat component: test values: image: @@ -232,18 +228,18 @@ data: enabled: false output.logstash: enabled: true - hosts: ["stx-logstash:5044"] + hosts: ["mon-logstash:5044"] output.elasticsearch: # this must be opposite of above output.logstash.enabled enabled: false - hosts: ["http://stx-elasticsearch-client:9200"] + hosts: ["http://mon-elasticsearch-client:9200"] ilm.pattern: "000001" processors: - add_kubernetes_metadata: in_cluster: true setup.kibana: # for on box kibana, note port 5601 is default - host: "stx-kibana" + host: "mon-kibana" setup.dashboards: enabled: true metricbeat.autodiscover: @@ -264,14 +260,14 @@ data: enabled: false output.logstash: enabled: true - hosts: ["stx-logstash:5044"] + hosts: ["mon-logstash:5044"] output.elasticsearch: enabled: false - hosts: ["http://stx-elasticsearch-client:9200"] + hosts: ["http://mon-elasticsearch-client:9200"] ilm.pattern: "000001" setup.kibana: # for onbox kibana, note port 5601 is default - host: "stx-kibana" + host: "mon-kibana" # for offbox kibana, will fail if we can't connect # host: "128.224.140.223" setup.dashboards: @@ -299,7 +295,7 @@ data: wait: timeout: 600 labels: - release: stx-kube-state-metrics + release: mon-kube-state-metrics test: enabled: false install: @@ -310,10 +306,10 @@ data: delete: - type: job labels: - release: stx-kube-state-metrics + release: mon-kube-state-metrics - type: pod labels: - release: stx-kube-state-metrics + release: mon-kube-state-metrics component: test values: image: @@ -338,7 +334,7 @@ data: wait: timeout: 600 labels: - release: stx-kibana + release: mon-kibana test: enabled: false install: @@ -349,10 +345,10 @@ data: delete: - type: job labels: - release: stx-kibana + release: mon-kibana - type: pod labels: - release: stx-kibana + release: mon-kibana component: test values: image: @@ -369,7 +365,7 @@ data: - "" files: kibana.yml: - elasticsearch.hosts: http://stx-elasticsearch-client:9200 + elasticsearch.hosts: http://mon-elasticsearch-client:9200 server.host: "::" nodeSelector: elastic-controller: "enabled" @@ -391,7 +387,7 @@ data: wait: timeout: 600 labels: - release: stx-logstash + release: mon-logstash test: enabled: false install: @@ -402,10 +398,10 @@ data: delete: - type: job labels: - release: stx-logstash + release: mon-logstash - type: pod labels: - release: stx-logstash + release: mon-logstash component: test values: image: @@ -414,7 +410,7 @@ data: nodeSelector: elastic-controller: "enabled" elasticsearch: - host: stx-elasticsearch-client + host: mon-elasticsearch-client config: elasticsearch.path: "" outputs: @@ -487,7 +483,7 @@ data: nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/ssl-redirect: "false" nginx.ingress.kubernetes.io/force-ssl-redirect: "false" - path: /stx-logstash(/|$)(.*) + path: /mon-logstash(/|$)(.*) servicePort: 9600 hosts: - "" @@ -573,7 +569,7 @@ metadata: schema: metadata/Document/v1 name: monitor-armada-manifest data: - release_prefix: stx + release_prefix: mon chart_groups: - nginx-ingress - kibana