[ceph-mon] Change configmap names to be based on release names

This change makes the ceph-mon configmap names dynamic based on release name to match how the ceph-osd chart is naming configmaps. The new ceph-mon post-apply job needs this in some cases in order not to have conflicting configmap names in separate releases. Change-Id: Id26d0a8310ccff80a608e25d2b0a74a41f9e6a55
2022-02-24 14:03:45 -07:00 · 2022-02-24 14:03:45 -07:00 · ad09539f71
commit ad09539f71
parent 2fc1ce4a14
7 changed files with 316 additions and 7 deletions
--- a/ceph-mon/Chart.yaml
+++ b/ceph-mon/Chart.yaml
@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph Mon
 name: ceph-mon
-version: 0.1.20
+version: 0.1.21
 home: https://github.com/ceph/ceph
 ...
--- a/ceph-mon/templates/configmap-bin.yaml
+++ b/ceph-mon/templates/configmap-bin.yaml
@ -18,7 +18,7 @@ limitations under the License.
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: ceph-mon-bin
+  name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }}
 data:
 {{- if .Values.images.local_registry.active }}
  image-repo-sync.sh: |
--- a/ceph-mon/templates/configmap-etc.yaml
+++ b/ceph-mon/templates/configmap-etc.yaml
@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}

-{{- define "ceph.configmap.etc" }}
+{{- define "ceph.mon.configmap.etc" }}
 {{- $configMapName := index . 0 }}
 {{- $envAll := index . 1 }}
 {{- with $envAll }}
@ -47,5 +47,5 @@ data:
 {{- end }}
 {{- end }}
 {{- if .Values.manifests.configmap_etc }}
-{{- list "ceph-mon-etc" . | include "ceph.configmap.etc" }}
+{{- list (printf "%s-%s" .Release.Name "etc") . | include "ceph.mon.configmap.etc" }}
 {{- end }}
--- a/ceph-mon/templates/daemonset-mon.yaml
+++ b/ceph-mon/templates/daemonset-mon.yaml
@ -15,7 +15,7 @@ limitations under the License.
 {{- if and .Values.manifests.daemonset_mon .Values.deployment.ceph }}
 {{- $envAll := . }}

-{{- $serviceAccountName := "ceph-mon" }}
+{{- $serviceAccountName := (printf "%s" .Release.Name) }}
 {{ tuple $envAll "mon" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@ -44,6 +44,14 @@ subjects:
  - kind: ServiceAccount
    name: {{ $serviceAccountName }}
    namespace: {{ $envAll.Release.Namespace }}
+{{- end }}
+
+{{- define "ceph.mon.daemonset" }}
+{{- $daemonset := index . 0 }}
+{{- $configMapName := index . 1 }}
+{{- $serviceAccountName := index . 2 }}
+{{- $envAll := index . 3 }}
+{{- with $envAll }}
 ---
 kind: DaemonSet
 apiVersion: apps/v1
@ -249,11 +257,11 @@ spec:
          emptyDir: {}
        - name: ceph-mon-bin
          configMap:
-            name: ceph-mon-bin
+            name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }}
            defaultMode: 0555
        - name: ceph-mon-etc
          configMap:
-            name: ceph-mon-etc
+            name: {{ $configMapName }}
            defaultMode: 0444
        - name: pod-var-lib-ceph
          hostPath:
@ -275,3 +283,13 @@ spec:
          secret:
            secretName: {{ .Values.secrets.keyrings.mds }}
 {{- end }}
+{{- end }}
+
+{{- if .Values.manifests.daemonset_mon }}
+{{- $daemonset := .Values.daemonset.prefix_name }}
+{{- $configMapName := (printf "%s-%s" .Release.Name "etc") }}
+{{- $serviceAccountName := (printf "%s" .Release.Name) }}
+{{- $daemonset_yaml := list $daemonset $configMapName $serviceAccountName . | include "ceph.mon.daemonset" | toString | fromYaml }}
+{{- $configmap_yaml := "ceph.mon.configmap.etc" }}
+{{- list $daemonset $daemonset_yaml $configmap_yaml $configMapName . | include "ceph.utils.mon_daemonset_overrides" }}
+{{- end }}
--- a/ceph-mon/templates/utils/_mon_daemonset_overrides.tpl
+++ b/ceph-mon/templates/utils/_mon_daemonset_overrides.tpl
@ -0,0 +1,287 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "ceph.utils.match_exprs_hash" }}
+  {{- $match_exprs := index . 0 }}
+  {{- $context := index . 1 }}
+  {{- $_ := set $context.Values "__match_exprs_hash_content" "" }}
+  {{- range $match_expr := $match_exprs }}
+    {{- $_ := set $context.Values "__match_exprs_hash_content" (print $context.Values.__match_exprs_hash_content $match_expr.key $match_expr.operator ($match_expr.values | quote)) }}
+  {{- end }}
+  {{- $context.Values.__match_exprs_hash_content | sha256sum | trunc 8 }}
+  {{- $_ := unset $context.Values "__match_exprs_hash_content" }}
+{{- end }}
+
+{{- define "ceph.utils.mon_daemonset_overrides" }}
+  {{- $daemonset := index . 0 }}
+  {{- $daemonset_yaml := index . 1 }}
+  {{- $configmap_include := index . 2 }}
+  {{- $configmap_name := index . 3 }}
+  {{- $context := index . 4 }}
+  {{- $_ := unset $context ".Files" }}
+  {{- $_ := set $context.Values "__daemonset_yaml" $daemonset_yaml }}
+  {{- $daemonset_root_name := printf "ceph_%s" $daemonset }}
+  {{- $_ := set $context.Values "__daemonset_list" list }}
+  {{- $_ := set $context.Values "__default" dict }}
+  {{- if hasKey $context.Values.conf "overrides" }}
+    {{- range $key, $val := $context.Values.conf.overrides }}
+
+      {{- if eq $key $daemonset_root_name }}
+        {{- range $type, $type_data := . }}
+
+          {{- if eq $type "hosts" }}
+            {{- range $host_data := . }}
+              {{/* dictionary that will contain all info needed to generate this
+              iteration of the daemonset */}}
+              {{- $current_dict := dict }}
+
+              {{/* set daemonset name */}}
+              {{- $_ := set $current_dict "name" $host_data.name }}
+
+              {{/* apply overrides */}}
+              {{- $override_conf_copy := $host_data.conf }}
+              {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}}
+              {{- $root_conf_copy := omit ($context.Values.conf | toYaml | fromYaml) "overrides" }}
+              {{- $merged_dict := mergeOverwrite $root_conf_copy $override_conf_copy }}
+              {{- $root_conf_copy2 := dict "conf" $merged_dict }}
+              {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "conf") "__daemonset_list" }}
+              {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }}
+              {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }}
+              {{- $_ := set $current_dict "nodeData" $root_conf_copy4 }}
+
+              {{/* Schedule to this host explicitly. */}}
+              {{- $nodeSelector_dict := dict }}
+
+              {{- $_ := set $nodeSelector_dict "key" "kubernetes.io/hostname" }}
+              {{- $_ := set $nodeSelector_dict "operator" "In" }}
+
+              {{- $values_list := list $host_data.name }}
+              {{- $_ := set $nodeSelector_dict "values" $values_list }}
+
+              {{- $list_aggregate := list $nodeSelector_dict }}
+              {{- $_ := set $current_dict "matchExpressions" $list_aggregate }}
+
+              {{/* store completed daemonset entry/info into global list */}}
+              {{- $list_aggregate := append $context.Values.__daemonset_list $current_dict }}
+              {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }}
+
+            {{- end }}
+          {{- end }}
+
+          {{- if eq $type "labels" }}
+            {{- $_ := set $context.Values "__label_list" . }}
+            {{- range $label_data := . }}
+              {{/* dictionary that will contain all info needed to generate this
+              iteration of the daemonset. */}}
+              {{- $_ := set $context.Values "__current_label" dict }}
+
+              {{/* set daemonset name */}}
+              {{- $_ := set $context.Values.__current_label "name" $label_data.label.key }}
+
+              {{/* apply overrides */}}
+              {{- $override_conf_copy := $label_data.conf }}
+              {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}}
+              {{- $root_conf_copy := omit ($context.Values.conf | toYaml | fromYaml) "overrides" }}
+              {{- $merged_dict := mergeOverwrite $root_conf_copy $override_conf_copy }}
+              {{- $root_conf_copy2 := dict "conf" $merged_dict }}
+              {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "conf") "__daemonset_list" }}
+              {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }}
+              {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }}
+              {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }}
+
+              {{/* Schedule to the provided label value(s) */}}
+              {{- $label_dict := omit $label_data.label "NULL" }}
+              {{- $_ := set $label_dict "operator" "In" }}
+              {{- $list_aggregate := list $label_dict }}
+              {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }}
+
+              {{/* Do not schedule to other specified labels, with higher
+              precedence as the list position increases. Last defined label
+              is highest priority. */}}
+              {{- $other_labels := without $context.Values.__label_list $label_data }}
+              {{- range $label_data2 := $other_labels }}
+                {{- $label_dict := omit $label_data2.label "NULL" }}
+
+                {{- $_ := set $label_dict "operator" "NotIn" }}
+
+                {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }}
+                {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }}
+              {{- end }}
+              {{- $_ := set $context.Values "__label_list" $other_labels }}
+
+              {{/* Do not schedule to any other specified hosts */}}
+              {{- range $type, $type_data := $val }}
+                {{- if eq $type "hosts" }}
+                  {{- range $host_data := . }}
+                    {{- $label_dict := dict }}
+
+                    {{- $_ := set $label_dict "key" "kubernetes.io/hostname" }}
+                    {{- $_ := set $label_dict "operator" "NotIn" }}
+
+                    {{- $values_list := list $host_data.name }}
+                    {{- $_ := set $label_dict "values" $values_list }}
+
+                    {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }}
+                    {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }}
+                  {{- end }}
+                {{- end }}
+              {{- end }}
+
+              {{/* store completed daemonset entry/info into global list */}}
+              {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }}
+              {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }}
+              {{- $_ := unset $context.Values "__current_label" }}
+
+            {{- end }}
+          {{- end }}
+        {{- end }}
+
+        {{/* scheduler exceptions for the default daemonset */}}
+        {{- $_ := set $context.Values.__default "matchExpressions" list }}
+
+        {{- range $type, $type_data := . }}
+          {{/* Do not schedule to other specified labels */}}
+          {{- if eq $type "labels" }}
+            {{- range $label_data := . }}
+              {{- $default_dict := omit $label_data.label "NULL" }}
+
+              {{- $_ := set $default_dict "operator" "NotIn" }}
+
+              {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }}
+              {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }}
+            {{- end }}
+          {{- end }}
+          {{/* Do not schedule to other specified hosts */}}
+          {{- if eq $type "hosts" }}
+            {{- range $host_data := . }}
+              {{- $default_dict := dict }}
+
+              {{- $_ := set $default_dict "key" "kubernetes.io/hostname" }}
+              {{- $_ := set $default_dict "operator" "NotIn" }}
+
+              {{- $values_list := list $host_data.name }}
+              {{- $_ := set $default_dict "values" $values_list }}
+
+              {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }}
+              {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }}
+            {{- end }}
+          {{- end }}
+        {{- end }}
+      {{- end }}
+    {{- end }}
+  {{- end }}
+
+  {{/* generate the default daemonset */}}
+
+  {{/* set name */}}
+  {{- $_ := set $context.Values.__default "name" "default" }}
+
+  {{/* no overrides apply, so copy as-is */}}
+  {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }}
+  {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }}
+  {{- $context_values := omit $context.Values "conf" }}
+  {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }}
+  {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }}
+  {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }}
+
+  {{/* add to global list */}}
+  {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }}
+  {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }}
+
+  {{- $_ := set $context.Values "__last_configmap_name" $configmap_name }}
+  {{- range $current_dict := $context.Values.__daemonset_list }}
+
+    {{- $context_novalues := omit $context "Values" }}
+    {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }}
+    {{- $_ := set $current_dict "nodeData" $merged_dict }}
+
+    {{/* name needs to be a DNS-1123 compliant name. Ensure lower case */}}
+    {{- $name_format1 := printf (print $daemonset_root_name "-" $current_dict.name) | lower }}
+    {{/* labels may contain underscores which would be invalid here, so we replace them with dashes
+    there may be other valid label names which would make for an invalid DNS-1123 name
+    but these will be easier to handle in future with sprig regex* functions
+    (not availabile in helm 2.5.1) */}}
+    {{- $name_format2 := $name_format1 | replace "_" "-" | replace "." "-" }}
+    {{/* To account for the case where the same label is defined multiple times in overrides
+    (but with different label values), we add a sha of the scheduling data to ensure
+    name uniqueness */}}
+    {{- $_ := set $current_dict "dns_1123_name" dict }}
+    {{- if hasKey $current_dict "matchExpressions" }}
+      {{- $_ := set $current_dict "dns_1123_name" (printf (print $name_format2 "-" (list $current_dict.matchExpressions $context | include "ceph.utils.match_exprs_hash"))) }}
+    {{- else }}
+      {{- $_ := set $current_dict "dns_1123_name" $name_format2 }}
+    {{- end }}
+
+    {{/* set daemonset metadata name */}}
+    {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }}
+    {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }}
+    {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }}
+
+    {{/* cross-reference configmap name to container volume definitions */}}
+    {{- $_ := set $context.Values "__volume_list" list }}
+    {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }}
+      {{- $_ := set $context.Values "__volume" $current_volume }}
+      {{- if hasKey $context.Values.__volume "configMap" }}
+        {{- if eq $context.Values.__volume.configMap.name $context.Values.__last_configmap_name }}
+          {{- $_ := set $context.Values.__volume.configMap "name" $current_dict.dns_1123_name }}
+        {{- end }}
+      {{- end }}
+      {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }}
+      {{- $_ := set $context.Values "__volume_list" $updated_list }}
+    {{- end }}
+    {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }}
+
+    {{/* populate scheduling restrictions */}}
+    {{- if hasKey $current_dict "matchExpressions" }}
+      {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }}
+      {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }}
+      {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }}
+      {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }}
+      {{- $match_exprs := dict }}
+      {{- $_ := set $match_exprs "matchExpressions" $current_dict.matchExpressions }}
+      {{- $appended_match_expr := list $match_exprs }}
+      {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $appended_match_expr }}
+    {{- end }}
+
+    {{/* input value hash for current set of values overrides */}}
+    {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }}
+    {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }}
+    {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }}
+    {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }}
+    {{- $cmap := list $current_dict.dns_1123_name $current_dict.nodeData | include $configmap_include }}
+    {{- $values_hash := $cmap | quote | sha256sum }}
+    {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_hash }}
+
+    {{/* generate configmap */}}
+---
+{{ $cmap }}
+
+    {{/* generate daemonset yaml */}}
+{{ range $k, $v := index $current_dict.nodeData.Values.conf.storage "mon" }}
+---
+{{- $_ := set $context.Values "__tmpYAML" dict }}
+
+{{ $dsNodeName := index $context.Values.__daemonset_yaml.metadata "name" }}
+{{ $localDsNodeName := print (trunc 54 $current_dict.dns_1123_name) "-" (print $dsNodeName $k | quote | sha256sum | trunc 8) }}
+{{- if not $context.Values.__tmpYAML.metadata }}{{- $_ := set $context.Values.__tmpYAML "metadata" dict }}{{- end }}
+{{- $_ := set $context.Values.__tmpYAML.metadata "name" $localDsNodeName }}
+
+{{ merge $context.Values.__tmpYAML $context.Values.__daemonset_yaml | toYaml }}
+
+{{ end }}
+
+---
+    {{- $_ := set $context.Values "__last_configmap_name" $current_dict.dns_1123_name }}
+  {{- end }}
+{{- end }}
--- a/ceph-mon/values.yaml
+++ b/ceph-mon/values.yaml
@ -296,6 +296,9 @@ conf:
    mon:
      directory: /var/lib/openstack-helm/ceph/mon

+daemonset:
+  prefix_name: "mon"
+
 dependencies:
  dynamic:
    common:
--- a/releasenotes/notes/ceph-mon.yaml
+++ b/releasenotes/notes/ceph-mon.yaml
@ -21,4 +21,5 @@ ceph-mon:
  - 0.1.18 Move ceph-mgr deployment to the ceph-mon chart
  - 0.1.19 Add a post-apply job to restart mons after mgrs
  - 0.1.20 Consolidate mon_endpoints discovery
+  - 0.1.21 Change configmap names to be based on release name
 ...