Elastic: Move to the elasticsearch charts at elastic/helm-charts

Story: 2005733 Task: 36962 Depends-On: https://review.opendev.org/687360 Change-Id: Idac62e3ebffba9ace2a2e247636c5726fd6f728d Signed-off-by: Kevin Smith <kevin.smith@windriver.com>
2019-10-07 10:49:00 -04:00 · 2019-10-07 10:49:00 -04:00 · 97576c7dc3
parent 79faff5044
commit 97576c7dc3
14 changed files with 420 additions and 63 deletions
--- a/1
+++ b/1
@ -1,2 +1,3 @@
 monitor-helm
+monitor-helm-elastic
 stx-monitor-helm
--- a/monitor-helm-elastic/Readme.rst
+++ b/monitor-helm-elastic/Readme.rst
@ -0,0 +1,10 @@
+This repo is for
+https://github.com/elastic/helm-charts/tree/master/elasticsearch
+
+Changes to this repo are needed for StarlingX and those changes are
+not yet merged.
+Rather than clone and diverge the repo, the repo is extracted at a particular
+git SHA, and patches are applied on top.
+
+As those patches are merged, the SHA can be updated and
+the local patches removed.
--- a/monitor-helm-elastic/centos/build_srpm.data
+++ b/monitor-helm-elastic/centos/build_srpm.data
@ -0,0 +1,8 @@
+TAR_NAME=helm-charts-elastic
+SHA=2bd7616ceddbdf2eee88965e2028ee37d304c79c
+VERSION=1.0.0
+TAR="$TAR_NAME-$SHA.tar.gz"
+
+COPY_LIST="${CGCS_BASE}/downloads/$TAR $PKG_BASE/files/* "
+
+TIS_PATCH_VER=1
--- a/monitor-helm-elastic/centos/monitor-helm-elastic.spec
+++ b/monitor-helm-elastic/centos/monitor-helm-elastic.spec
@ -0,0 +1,75 @@
+%global sha 2bd7616ceddbdf2eee88965e2028ee37d304c79c
+%global helm_folder  /usr/lib/helm
+%global helmchart_version 0.1.0
+%global _default_patch_flags --no-backup-if-mismatch --prefix=/tmp/junk
+
+Summary: Monitor-Helm-Elastic charts
+Name: monitor-helm-elastic
+Version: 1.0
+Release: %{tis_patch_ver}%{?_tis_dist}
+License: Apache-2.0
+Group: base
+Packager: Wind River <info@windriver.com>
+URL: https://github.com/elastic/helm-charts/
+
+Source0: helm-charts-elastic-%{sha}.tar.gz
+Source1: repositories.yaml
+Source2: index.yaml
+
+BuildArch:     noarch
+
+Patch01: 0001-add-makefile.patch
+Patch02: 0002-Add-compatibility-for-k8s-1.16.patch
+Patch03: 0003-use-oss-image.patch
+
+BuildRequires: helm
+
+%description
+Monitor Helm elasticsearch charts
+
+%prep
+%setup -n helm-charts-elastic
+%patch01 -p1
+%patch02 -p1
+%patch03 -p1
+
+
+%build
+# initialize helm and build the toolkit
+# helm init --client-only does not work if there is no networking
+# The following commands do essentially the same as: helm init
+%define helm_home %{getenv:HOME}/.helm
+mkdir %{helm_home}
+mkdir %{helm_home}/repository
+mkdir %{helm_home}/repository/cache
+mkdir %{helm_home}/repository/local
+mkdir %{helm_home}/plugins
+mkdir %{helm_home}/starters
+mkdir %{helm_home}/cache
+mkdir %{helm_home}/cache/archive
+
+# Stage a repository file that only has a local repo
+cp %{SOURCE1} %{helm_home}/repository/repositories.yaml
+
+# Stage a local repo index that can be updated by the build
+cp %{SOURCE2} %{helm_home}/repository/local/index.yaml
+
+# Host a server for the charts
+helm serve --repo-path . &
+helm repo rm local
+helm repo add local http://localhost:8879/charts
+
+# Create the tgz files
+rm elasticsearch/Makefile
+make elasticsearch
+
+# terminate helm server (the last backgrounded task)
+kill %1
+
+%install
+install -d -m 755 ${RPM_BUILD_ROOT}%{helm_folder}
+install -p -D -m 755 *.tgz ${RPM_BUILD_ROOT}%{helm_folder}
+
+%files
+%defattr(-,root,root,-)
+%{helm_folder}/*
--- a/monitor-helm-elastic/files/0001-add-makefile.patch
+++ b/monitor-helm-elastic/files/0001-add-makefile.patch
@ -0,0 +1,62 @@
+From c9c3e8ff214360eb2e5d9e7728b7bee8fe771eea Mon Sep 17 00:00:00 2001
+From: Kevin Smith <kevin.smith@windriver.com>
+Date: Tue, 1 Oct 2019 15:25:32 -0400
+Subject: [PATCH 1/1] add makefile
+
+---
+ Makefile | 43 +++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 43 insertions(+)
+ create mode 100644 Makefile
+
+diff --git a/Makefile b/Makefile
+new file mode 100644
+index 0000000..5cf4447
+--- /dev/null
+++ b/Makefile
+@@ -0,0 +1,43 @@
+#
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# It's necessary to set this because some environments don't link sh -> bash.
+SHELL := /bin/bash
+TASK  := build
+
+EXCLUDES := helm-toolkit doc tests tools logs tmp
+CHARTS := helm-toolkit $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.)))
+
+.PHONY: $(EXCLUDES) $(CHARTS)
+
+all: $(CHARTS)
+
+$(CHARTS):
+	@if [ -d $@ ]; then \
+		echo; \
+		echo "===== Processing [$@] chart ====="; \
+		make $(TASK)-$@; \
+	fi
+
+init-%:
+	if [ -f $*/Makefile ]; then make -C $*; fi
+	if [ -f $*/requirements.yaml ]; then helm dep up $*; fi
+
+lint-%: init-%
+	if [ -d $* ]; then helm lint $*; fi
+
+build-%: lint-%
+	if [ -d $* ]; then helm package $*; fi
+
+clean:
+	@echo "Clean all build artifacts"
+	rm -f */templates/_partials.tpl */templates/_globals.tpl
+	rm -f *tgz */charts/*tgz */requirements.lock
+	rm -rf */charts */tmpcharts
+
+%:
+	@:
+-- 
+1.8.3.1
+
--- a/monitor-helm-elastic/files/0002-Add-compatibility-for-k8s-1.16.patch
+++ b/monitor-helm-elastic/files/0002-Add-compatibility-for-k8s-1.16.patch
@ -0,0 +1,68 @@
+From f79169b8e6a57881952ca4d2c338eeda179471e9 Mon Sep 17 00:00:00 2001
+From: Kevin Smith <kevin.smith@windriver.com>
+Date: Mon, 7 Oct 2019 11:18:05 -0400
+Subject: [PATCH 1/1] Add compatibility for k8s 1.16
+
+---
+ elasticsearch/templates/_helpers.tpl     | 22 ++++++++++++++++++++++
+ elasticsearch/templates/ingress.yaml     |  2 +-
+ elasticsearch/templates/statefulset.yaml |  2 +-
+ 3 files changed, 24 insertions(+), 2 deletions(-)
+
+diff --git a/elasticsearch/templates/_helpers.tpl b/elasticsearch/templates/_helpers.tpl
+index d651d56..ccfd8c3 100755
+--- a/elasticsearch/templates/_helpers.tpl
+++ b/elasticsearch/templates/_helpers.tpl
+@@ -47,3 +47,25 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
+   {{- end -}}
+ {{- end -}}
+ {{- end -}}
+
+{{/*
+Return the appropriate apiVersion for statefulset.
+*/}}
+{{- define "elasticsearch.statefulset.apiVersion" -}}
+{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.GitVersion -}}
+{{- print "apps/v1beta2" -}}
+{{- else -}}
+{{- print "apps/v1" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the appropriate apiVersion for ingress.
+*/}}
+{{- define "elasticsearch.ingress.apiVersion" -}}
+{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+{{- print "extensions/v1beta1" -}}
+{{- else -}}
+{{- print "networking.k8s.io/v1beta1" -}}
+{{- end -}}
+{{- end -}}
+diff --git a/elasticsearch/templates/ingress.yaml b/elasticsearch/templates/ingress.yaml
+index 39167ca..1715b97 100644
+--- a/elasticsearch/templates/ingress.yaml
+++ b/elasticsearch/templates/ingress.yaml
+@@ -2,7 +2,7 @@
+ {{- $fullName := include "uname" . -}}
+ {{- $servicePort := .Values.httpPort -}}
+ {{- $ingressPath := .Values.ingress.path -}}
+-apiVersion: extensions/v1beta1
+apiVersion: {{ template "elasticsearch.ingress.apiVersion" . }}
+ kind: Ingress
+ metadata:
+   name: {{ $fullName }}
+diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml
+index 004578f..36c7af6 100644
+--- a/elasticsearch/templates/statefulset.yaml
+++ b/elasticsearch/templates/statefulset.yaml
+@@ -1,5 +1,5 @@
+ ---
+-apiVersion: apps/v1beta1
+apiVersion: {{ template "elasticsearch.statefulset.apiVersion" . }}
+ kind: StatefulSet
+ metadata:
+   name: {{ template "uname" . }}
+-- 
+1.8.3.1
+
--- a/monitor-helm-elastic/files/0003-use-oss-image.patch
+++ b/monitor-helm-elastic/files/0003-use-oss-image.patch
@ -0,0 +1,25 @@
+From a1ea0a2f488660f9cfbab44a4fb8b163e135ebf5 Mon Sep 17 00:00:00 2001
+From: Kevin Smith <kevin.smith@windriver.com>
+Date: Tue, 8 Oct 2019 13:35:57 -0400
+Subject: [PATCH 1/1] use oss image
+
+---
+ elasticsearch/values.yaml | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml
+index ccab462..f4fca1f 100755
+--- a/elasticsearch/values.yaml
+++ b/elasticsearch/values.yaml
+@@ -42,7 +42,7 @@ secretMounts: []
+ #    secretName: elastic-certificates
+ #    path: /usr/share/elasticsearch/config/certs
+ 
+-image: "docker.elastic.co/elasticsearch/elasticsearch"
+image: "docker.elastic.co/elasticsearch/elasticsearch-oss"
+ imageTag: "7.3.2"
+ imagePullPolicy: "IfNotPresent"
+ 
+-- 
+1.8.3.1
+
--- a/monitor-helm-elastic/files/index.yaml
+++ b/monitor-helm-elastic/files/index.yaml
@ -0,0 +1,4 @@
+---
+apiVersion: v1
+entries: {}
+generated: 2019-01-07T12:33:46.098166523-06:00
--- a/monitor-helm-elastic/files/repositories.yaml
+++ b/monitor-helm-elastic/files/repositories.yaml
@ -0,0 +1,12 @@
+---
+apiVersion: v1
+generated: 2019-01-02T15:19:36.215111369-06:00
+repositories:
+  - caFile: ""
+    cache: /builddir/.helm/repository/cache/local-index.yaml
+    certFile: ""
+    keyFile: ""
+    name: local
+    password: ""
+    url: http://127.0.0.1:8879/charts
+    username: ""
--- a/monitor-helm/Readme.rst
+++ b/monitor-helm/Readme.rst
@ -1,5 +1,4 @@
 This repo is for
-https://github.com/helm/charts/tree/master/stable/elasticsearch
 https://github.com/helm/charts/tree/master/stable/filebeat
 https://github.com/helm/charts/tree/master/stable/metricbeat
 https://github.com/helm/charts/tree/master/stable/kibana
--- a/monitor-helm/centos/build_srpm.data
+++ b/monitor-helm/centos/build_srpm.data
@ -5,4 +5,4 @@ TAR="$TAR_NAME-$SHA.tar.gz"

 COPY_LIST="${CGCS_BASE}/downloads/$TAR $PKG_BASE/files/* "

-TIS_PATCH_VER=2
+TIS_PATCH_VER=3
--- a/monitor-helm/centos/monitor-helm.spec
+++ b/monitor-helm/centos/monitor-helm.spec
@ -70,7 +70,6 @@ helm repo add local http://localhost:8879/charts

 # Create the tgz files
 cd stable
-make elasticsearch
 make filebeat
 make metricbeat
 make kube-state-metrics
--- a/stx-monitor-helm/centos/stx-monitor-helm.spec
+++ b/stx-monitor-helm/centos/stx-monitor-helm.spec
@ -13,7 +13,9 @@ Source0: %{name}-%{version}.tar.gz

 BuildArch: noarch
 BuildRequires: monitor-helm
+BuildRequires: monitor-helm-elastic
 Requires: monitor-helm
+Requires: monitor-helm-elastic

 %description
 StarlingX Monitor Application Armada Helm Charts
--- a/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml
+++ b/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml
@ -54,68 +54,158 @@ data:
  dependencies: []
 ---
 schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: elasticsearch
 data:
-  chart_name: elasticsearch
-  release: elasticsearch
-  namespace: monitor
-  wait:
-    timeout: 600
-    labels:
-      release: mon-elasticsearch
-  test:
-    enabled: false
+  chart_name: elasticsearch-data
+  dependencies: []
  install:
    no_hooks: false
+  namespace: monitor
+  release: elasticsearch-data
+  source:
+    location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-7.3.2.tgz
+    reference: master
+    subpath: elasticsearch
+    type: tar
+  test:
+    enabled: false
  upgrade:
    no_hooks: false
    pre:
      delete:
-        - type: job
-          labels:
-            release: mon-elasticsearch
-        - type: pod
-          labels:
-            release: mon-elasticsearch
-            component: test
+      - labels:
+          release: mon-elasticsearch-data
+        type: job
+      - labels:
+          component: test
+          release: mon-elasticsearch-data
+        type: pod
  values:
-    image:
-      repository: "docker.elastic.co/elasticsearch/elasticsearch-oss"
-      tag: 7.2.0
-    cluster:
-      env:
-        MINIMUM_MASTER_NODES: "1"
-        EXPECTED_MASTER_NODES: "1"
-        RECOVER_AFTER_MASTER_NODES: "1"
-    client:
-      ingress:
-        enabled: true
-        annotations:
-          kubernetes.io/ingress.class: "nginx"
-          nginx.ingress.kubernetes.io/rewrite-target: /$2
-          nginx.ingress.kubernetes.io/ssl-redirect: "false"
-          nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
-        # Note in order to differentiate from kibana, prefix with
-        # mon-elasticsearch-client, but strip before presenting
-        # to elasticsearch with above rewrite-target
-        path: /mon-elasticsearch-client(/|$)(.*)
-        hosts:
-          - ""
-    master:
-      updateStrategy:
-        type: "RollingUpdate"
-    data:
-      terminationGracePeriodSeconds: 240
-      updateStrategy:
-        type: "RollingUpdate"
-  source:
-    type: tar
-    location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-1.24.0.tgz
-    subpath: elasticsearch
-    reference: master
+    clusterName: 'mon-elasticsearch'
+    nodeGroup: 'data'
+    roles:
+      master: 'false'
+      data: 'true'
+      ingest: 'false'
+    replicas: 2
+    minimumMasterNodes: 1
+    nodeSelector:
+      elastic-data: enabled
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss
+    imageTag: 7.3.2
+    esMajorVersion: 7
+  wait:
+    labels:
+      release: mon-elasticsearch-data
+    timeout: 600
+metadata:
+  name: elasticsearch-data
+  schema: metadata/Document/v1
+---
+schema: armada/Chart/v1
+data:
+  chart_name: elasticsearch-client
  dependencies: []
+  install:
+    no_hooks: false
+  namespace: monitor
+  release: elasticsearch-client
+  source:
+    location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-7.3.2.tgz
+    reference: master
+    subpath: elasticsearch
+    type: tar
+  test:
+    enabled: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+      - labels:
+          release: mon-elasticsearch-client
+        type: job
+      - labels:
+          component: test
+          release: mon-elasticsearch-client
+        type: pod
+  values:
+    clusterName: 'mon-elasticsearch'
+    nodeGroup: 'client'
+    roles:
+      master: 'false'
+      data: 'false'
+      ingest: 'true'
+    replicas: 2
+    minimumMasterNodes: 1
+    ingress:
+      enabled: true
+      annotations:
+        kubernetes.io/ingress.class: nginx
+        nginx.ingress.kubernetes.io/force-ssl-redirect: 'false'
+        nginx.ingress.kubernetes.io/rewrite-target: /$2
+        nginx.ingress.kubernetes.io/ssl-redirect: 'false'
+      hosts:
+      - ''
+      path: /mon-elasticsearch-client(/|$)(.*)
+    nodeSelector:
+      elastic-client: enabled
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss
+    imageTag: 7.3.2
+    esMajorVersion: 7
+  wait:
+    labels:
+      release: mon-elasticsearch-client
+    timeout: 600
+metadata:
+  name: elasticsearch-client
+  schema: metadata/Document/v1
+---
+schema: armada/Chart/v1
+data:
+  chart_name: elasticsearch-master
+  dependencies: []
+  install:
+    no_hooks: false
+  namespace: monitor
+  release: elasticsearch-master
+  source:
+    location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-7.3.2.tgz
+    reference: master
+    subpath: elasticsearch
+    type: tar
+  test:
+    enabled: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+      - labels:
+          release: mon-elasticsearch-master
+        type: job
+      - labels:
+          component: test
+          release: mon-elasticsearch-master
+        type: pod
+  values:
+    clusterName: 'mon-elasticsearch'
+    nodeGroup: 'master'
+    roles:
+      master: 'true'
+      data: 'false'
+      ingest: 'false'
+    replicas: 1
+    minimumMasterNodes: 1
+    nodeSelector:
+      elastic-master: enabled
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss
+    imageTag: 7.3.2
+    esMajorVersion: 7
+  wait:
+    labels:
+      release: mon-elasticsearch-master
+    timeout: 600
+metadata:
+  name: elasticsearch-master
+  schema: metadata/Document/v1
 ---
 schema: armada/Chart/v1
 metadata:
@ -147,7 +237,7 @@ data:
  values:
    image:
      repository: "docker.elastic.co/beats/filebeat-oss"
-      tag: 7.2.0
+      tag: 7.3.2
    config:
      output.file:
        enabled: false
@ -225,7 +315,7 @@ data:
  values:
    image:
      repository: "docker.elastic.co/beats/metricbeat-oss"
-      tag: 7.2.0
+      tag: 7.3.2
    daemonset:
      config:
        output.file:
@ -365,7 +455,7 @@ data:
  values:
    image:
      repository: "docker.elastic.co/kibana/kibana-oss"
-      tag: 7.2.0
+      tag: 7.3.2
    ingress:
      enabled: true
      annotations:
@ -418,7 +508,7 @@ data:
  values:
    image:
      repository: "docker.elastic.co/logstash/logstash-oss"
-      tag: 7.2.0
+      tag: 7.3.2
    nodeSelector:
      elastic-controller: "enabled"
    elasticsearch:
@ -518,13 +608,15 @@ data:
 ---
 schema: armada/ChartGroup/v1
 metadata:
-  schema: metadata/Document/v1
  name: elasticsearch
+  schema: metadata/Document/v1
 data:
+  chart_group:
+  - elasticsearch-master
+  - elasticsearch-data
+  - elasticsearch-client
  description: "Deploy elasticsearch"
  sequenced: true
-  chart_group:
-    - elasticsearch
 ---
 schema: armada/ChartGroup/v1
 metadata: