Build helm-toolkit for armada to decouple distro from flock

This provides helm-toolkit identically to how openstack-helm-infra
is built in repo stx/openstack-armada-app. This version of
helm-toolkit is used to build armada chart. This decouples distro
build from flock.

Story: 2007000
Task: 38893

Change-Id: I537625236fb05200e5380c4f23e3e144e24c8675
Signed-off-by: Jim Gauld <james.gauld@windriver.com>

centos_pkg_dirs

@@ -59,6 +59,7 @@ kubernetes/docker-distribution
 kubernetes/etcd
 kubernetes/helm
 kubernetes/chartmuseum
+kubernetes/armada-helm-toolkit
 kubernetes/armada
 grub/grubby
 base/dpkg

centos_tarball-dl.lst

@@ -39,6 +39,7 @@ lldpd-0.9.0.tar.gz#lldpd-0.9.0#https://media.luffy.cx/files/lldpd/lldpd-0.9.0.ta
 lua-1fce39c6397056db645718b8f5821571d97869a4.tar.gz#lua#https://api.github.com/repos/ceph/lua/tarball/1fce39c6397056db645718b8f5821571d97869a4#https##
 !mariadb-10.1.28.tar.gz#mariadb-10.1.28#https://github.com/MariaDB/server/archive/mariadb-10.1.28.tar.gz#http_script##post-dl-script/mariadb.sh
 !MLNX_OFED_SRC-4.7-3.2.9.0.tgz#MLNX_OFED_SRC-4.7-3.2.9.0#http://content.mellanox.com/ofed/MLNX_OFED-4.7-3.2.9.0/MLNX_OFED_SRC-4.7-3.2.9.0.tgz#http_script##post-dl-script/MLNX_OFED.sh
+openstack-helm-infra-c9d6676bf9a5aceb311dc31dadd07cba6a3d6392.tar.gz#openstack-helm-infra#https://github.com/openstack/openstack-helm-infra/archive/c9d6676bf9a5aceb311dc31dadd07cba6a3d6392.tar.gz#http##
 openvswitch-2.11.0.tar.gz#openvswitch-2.11.0#https://github.com/openvswitch/ovs/archive/v2.11.0.tar.gz#http##
 puppet-boolean-22b726dd78b0a60a224cc7054aebbf28e9306f62.tar.gz#puppet-boolean#https://github.com/voxpupuli/puppet-boolean/tarball/22b726dd78b0a60a224cc7054aebbf28e9306f62#http##
 puppet-dnsmasq-cff07e90890662972c97684a2baee964f68ff3ed.tar.gz#packstack/puppet/modules/dnsmasq/#https://github.com/procore/puppet-dnsmasq/tarball/cff07e90890662972c97684a2baee964f68ff3ed#http##

kubernetes/armada-helm-toolkit/Readme.rst

@@ -0,0 +1,8 @@
+This repo is for https://github.com/openstack/openstack-helm-infra
+
+Changes to this repo are needed for StarlingX and those changes are
+not yet merged.
+Rather than clone and diverge the repo, the repo is extracted at a particular
+git SHA, and patches are applied on top.
+
+As those patches are merged, the SHA can be updated and the local patches removed.

kubernetes/armada-helm-toolkit/centos/armada-helm-toolkit.spec

@@ -0,0 +1,61 @@
+%global src_name openstack-helm-infra
+%global sha c9d6676bf9a5aceb311dc31dadd07cba6a3d6392
+%global helm_folder  /usr/lib/helm
+
+Summary: Openstack-Helm-Infra helm-toolkit chart
+Name: armada-helm-toolkit
+Version: 1.0
+Release: %{tis_patch_ver}%{?_tis_dist}
+License: Apache-2.0
+Group: base
+Packager: Wind River <info@windriver.com>
+URL: https://github.com/openstack/openstack-helm-infra
+
+Source0: %{src_name}-%{sha}.tar.gz
+
+BuildArch:     noarch
+
+# Note patches 0003, 0005, 0007 through 0013 do not apply to helm-toolkit
+Patch01: 0001-Allow-multiple-containers-per-daemonset-pod.patch
+Patch02: 0002-Add-imagePullSecrets-in-service-account.patch
+Patch04: 0004-Partial-revert-of-31e3469d28858d7b5eb6355e88b6f49fd6.patch
+Patch06: 0006-Fix-pod-restarts-on-all-workers-when-worker-added-re.patch
+
+BuildRequires: helm
+BuildRequires: chartmuseum
+
+%description
+Openstack Helm Infra helm-toolkit chart
+
+%prep
+%setup -n openstack-helm-infra
+%patch01 -p1
+%patch02 -p1
+%patch04 -p1
+%patch06 -p1
+
+
+%build
+# Host a server for the charts
+chartmuseum --debug --port=8879 --context-path='/charts' --storage="local" --storage-local-rootdir="." &
+sleep 2
+helm repo add local http://localhost:8879/charts
+
+# Make the charts. These produce tgz files
+make helm-toolkit
+# Both armada-helm-toolkit and openstack-helm-infra provide the same
+# helm-toolkit tarball filename. Rename files with 'armada-' prefix
+# to prevent 'Transaction check error'.
+for filename in *.tgz; do mv -v "$filename" "armada-$filename"; done
+
+# terminate helm server (the last backgrounded task)
+kill %1
+
+%install
+install -d -m 755 ${RPM_BUILD_ROOT}%{helm_folder}
+install -p -D -m 755 *.tgz ${RPM_BUILD_ROOT}%{helm_folder}
+
+%files
+%dir %attr(0755,root,root) %{helm_folder}
+%defattr(-,root,root,-)
+%{helm_folder}/*

kubernetes/armada-helm-toolkit/centos/build_srpm.data

@@ -0,0 +1,11 @@
+# NOTE: This package and version is identical to
+# stx/openstack-armada-app/openstack-helm-infra .
+# This decouples distro build of armada chart from flock.
+TAR_NAME=openstack-helm-infra
+SHA=c9d6676bf9a5aceb311dc31dadd07cba6a3d6392
+VERSION=1.0.0
+TAR="$TAR_NAME-$SHA.tar.gz"
+
+COPY_LIST="${CGCS_BASE}/downloads/$TAR $PKG_BASE/files/*"
+
+TIS_PATCH_VER=0

kubernetes/armada-helm-toolkit/files/0001-Allow-multiple-containers-per-daemonset-pod.patch

@@ -0,0 +1,40 @@
+From 47315e28d44cff586f6fff026dd00e61c2c77bcd Mon Sep 17 00:00:00 2001
+From: Gerry Kopec <Gerry.Kopec@windriver.com>
+Date: Wed, 9 Jan 2019 20:11:33 -0500
+Subject: [PATCH 1/4] Allow multiple containers per daemonset pod
+
+Remove code that restricted daemonset pods to single containers.
+Container names will default to name from helm chart template.
+Required for nova cold migrations to work.
+
+Story: 2003876
+Task: 26735
+Change-Id: Icce660415d43baefbbf768a785c5dedf04ea2930
+Signed-off-by: Gerry Kopec <Gerry.Kopec@windriver.com>
+(cherry picked from commit 7ca30319f418cd39db5ecf44cce5fb5fe39c458e)
+Signed-off-by: Robert Church <robert.church@windriver.com>
+---
+ helm-toolkit/templates/utils/_daemonset_overrides.tpl | 7 -------
+ 1 file changed, 7 deletions(-)
+
+diff --git a/helm-toolkit/templates/utils/_daemonset_overrides.tpl b/helm-toolkit/templates/utils/_daemonset_overrides.tpl
+index e352bc9..10ab166 100644
+--- a/helm-toolkit/templates/utils/_daemonset_overrides.tpl
++++ b/helm-toolkit/templates/utils/_daemonset_overrides.tpl
+@@ -225,13 +225,6 @@ limitations under the License.
+     {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }}
+     {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }}
+ 
+-    {{/* set container name
+-    assume not more than one container is defined */}}
+-    {{- $container := first $context.Values.__daemonset_yaml.spec.template.spec.containers }}
+-    {{- $_ := set $container "name" $current_dict.dns_1123_name }}
+-    {{- $cont_list := list $container }}
+-    {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "containers" $cont_list }}
+-
+     {{/* cross-reference configmap name to container volume definitions */}}
+     {{- $_ := set $context.Values "__volume_list" list }}
+     {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }}
+-- 
+2.7.4
+

kubernetes/armada-helm-toolkit/files/0002-Add-imagePullSecrets-in-service-account.patch

@@ -0,0 +1,26 @@
+From ac3f9db5ac1a19af71136752f5709ba1da55d201 Mon Sep 17 00:00:00 2001
+From: Angie Wang <angie.wang@windriver.com>
+Date: Mon, 11 Feb 2019 11:29:03 -0500
+Subject: [PATCH 2/4] Add imagePullSecrets in service account
+
+Signed-off-by: Robert Church <robert.church@windriver.com>
+---
+ helm-toolkit/templates/snippets/_kubernetes_pod_rbac_serviceaccount.tpl | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_serviceaccount.tpl b/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_serviceaccount.tpl
+index b4cf1a6..2f4113b 100644
+--- a/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_serviceaccount.tpl
++++ b/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_serviceaccount.tpl
+@@ -44,6 +44,8 @@ kind: ServiceAccount
+ metadata:
+   name: {{ $saName }}
+   namespace: {{ $saNamespace }}
++imagePullSecrets:
++  - name: default-registry-key
+ {{- range $k, $v := $deps -}}
+ {{- if eq $k "services" }}
+ {{- range $serv := $v }}
+-- 
+2.16.5
+

kubernetes/armada-helm-toolkit/files/0004-Partial-revert-of-31e3469d28858d7b5eb6355e88b6f49fd6.patch

@@ -0,0 +1,65 @@
+From b3829fef30e76fdf498fa1d0d35185f642dce5f6 Mon Sep 17 00:00:00 2001
+From: Robert Church <robert.church@windriver.com>
+Date: Mon, 8 Apr 2019 02:12:39 -0400
+Subject: [PATCH 4/4] Partial revert of
+ 31e3469d28858d7b5eb6355e88b6f49fd62032be
+
+Suspect that new use of mergeOverwrite vs. merge is breaking the
+per-host DaemonSet overrides.
+
+Signed-off-by: Robert Church <robert.church@windriver.com>
+---
+ helm-toolkit/templates/utils/_daemonset_overrides.tpl | 12 ++++++------
+ 1 file changed, 6 insertions(+), 6 deletions(-)
+
+diff --git a/helm-toolkit/templates/utils/_daemonset_overrides.tpl b/helm-toolkit/templates/utils/_daemonset_overrides.tpl
+index 10ab166..ab1177a 100644
+--- a/helm-toolkit/templates/utils/_daemonset_overrides.tpl
++++ b/helm-toolkit/templates/utils/_daemonset_overrides.tpl
+@@ -49,10 +49,10 @@ limitations under the License.
+               {{- $override_conf_copy := $host_data.conf }}
+               {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}}
+               {{- $root_conf_copy := omit ($context.Values.conf | toYaml | fromYaml) "overrides" }}
+-              {{- $merged_dict := mergeOverwrite $root_conf_copy $override_conf_copy }}
++              {{- $merged_dict := merge $override_conf_copy $root_conf_copy }}
+               {{- $root_conf_copy2 := dict "conf" $merged_dict }}
+               {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "conf") "__daemonset_list" }}
+-              {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }}
++              {{- $root_conf_copy3 := merge $context_values $root_conf_copy2 }}
+               {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }}
+               {{- $_ := set $current_dict "nodeData" $root_conf_copy4 }}
+ 
+@@ -89,10 +89,10 @@ limitations under the License.
+               {{- $override_conf_copy := $label_data.conf }}
+               {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}}
+               {{- $root_conf_copy := omit ($context.Values.conf | toYaml | fromYaml) "overrides" }}
+-              {{- $merged_dict := mergeOverwrite $root_conf_copy $override_conf_copy }}
++              {{- $merged_dict := merge $override_conf_copy $root_conf_copy }}
+               {{- $root_conf_copy2 := dict "conf" $merged_dict }}
+               {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "conf") "__daemonset_list" }}
+-              {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }}
++              {{- $root_conf_copy3 := merge $context_values $root_conf_copy2 }}
+               {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }}
+               {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }}
+ 
+@@ -187,7 +187,7 @@ limitations under the License.
+   {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }}
+   {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }}
+   {{- $context_values := omit $context.Values "conf" }}
+-  {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }}
++  {{- $root_conf_copy3 := merge $context_values $root_conf_copy2 }}
+   {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }}
+   {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }}
+ 
+@@ -198,7 +198,7 @@ limitations under the License.
+   {{- range $current_dict := $context.Values.__daemonset_list }}
+ 
+     {{- $context_novalues := omit $context "Values" }}
+-    {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }}
++    {{- $merged_dict := merge $current_dict.nodeData $context_novalues }}
+     {{- $_ := set $current_dict "nodeData" $merged_dict }}
+     {{/* Deep copy original daemonset_yaml */}}
+     {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }}
+-- 
+2.7.4
+

kubernetes/armada-helm-toolkit/files/0006-Fix-pod-restarts-on-all-workers-when-worker-added-re.patch

@@ -0,0 +1,46 @@
+From 326fcd76f54d7c099f4c3da6c31eefe0eef2e236 Mon Sep 17 00:00:00 2001
+From: Ovidiu Poncea <ovidiu.poncea@windriver.com>
+Date: Mon, 29 Jul 2019 08:00:01 -0400
+Subject: [PATCH] Fix pod restarts on all workers when worker added/removed
+
+---
+ helm-toolkit/templates/utils/_daemonset_overrides.tpl | 4 ++--
+ helm-toolkit/templates/utils/_hash.tpl                | 2 +-
+ 2 files changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/helm-toolkit/templates/utils/_daemonset_overrides.tpl b/helm-toolkit/templates/utils/_daemonset_overrides.tpl
+index ab1177a..e564869 100644
+--- a/helm-toolkit/templates/utils/_daemonset_overrides.tpl
++++ b/helm-toolkit/templates/utils/_daemonset_overrides.tpl
+@@ -215,7 +215,7 @@ limitations under the License.
+     name uniqueness */}}
+     {{- $_ := set $current_dict "dns_1123_name" dict }}
+     {{- if hasKey $current_dict "matchExpressions" }}
+-      {{- $_ := set $current_dict "dns_1123_name" (printf (print $name_format2 "-" ($current_dict.matchExpressions | quote | sha256sum | trunc 8))) }}
++      {{- $_ := set $current_dict "dns_1123_name" (printf (print $name_format2 "-" ($current_dict.matchExpressions | toJson | sha256sum | trunc 8))) }}
+     {{- else }}
+       {{- $_ := set $current_dict "dns_1123_name" $name_format2 }}
+     {{- end }}
+@@ -258,7 +258,7 @@ limitations under the License.
+     {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }}
+     {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }}
+     {{- $cmap := list $current_dict.dns_1123_name $current_dict.nodeData | include $configmap_include }}
+-    {{- $values_hash := $cmap | quote | sha256sum }}
++    {{- $values_hash := $cmap | toJson | sha256sum }}
+     {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_hash }}
+ 
+     {{/* generate configmap */}}
+diff --git a/helm-toolkit/templates/utils/_hash.tpl b/helm-toolkit/templates/utils/_hash.tpl
+index 1041ec0..e419e3b 100644
+--- a/helm-toolkit/templates/utils/_hash.tpl
++++ b/helm-toolkit/templates/utils/_hash.tpl
+@@ -19,5 +19,5 @@ limitations under the License.
+ {{- $context := index . 1 -}}
+ {{- $last := base $context.Template.Name }}
+ {{- $wtf := $context.Template.Name | replace $last $name -}}
+-{{- include $wtf $context | sha256sum | quote -}}
++{{- include $wtf $context | toJson | sha256sum | quote -}}
+ {{- end -}}
+-- 
+2.7.4
+

kubernetes/armada/centos/armada.spec

@@ -20,7 +20,7 @@ Patch1:         0001-Add-Helm-v2-client-initialization-using-tiller-postS.patch
 BuildArch:      noarch
 
 BuildRequires: helm
-BuildRequires: openstack-helm-infra
+BuildRequires: armada-helm-toolkit
 BuildRequires: chartmuseum
 
 %description
@@ -55,7 +55,7 @@ BuildRequires: chartmuseum
 #   (we don't need to create them)
 
 # Stage helm-toolkit in the local repo
-cp %{helm_folder}/helm-toolkit-%{toolkit_version}.tgz %{charts_staging}
+cp %{helm_folder}/armada-helm-toolkit-%{toolkit_version}.tgz %{charts_staging}/helm-toolkit-%{toolkit_version}.tgz
 
 # Host a local server for the charts.
 chartmuseum --debug --port=8879 --context-path='/charts' --storage="local" --storage-local-rootdir="%{charts_staging}" &