[CEPH] Discovering ceph-mon endpoints

This is a code improvement to reuse ceph monitor doscovering function
in different templates. Calling the mentioned above function from
a single place (helm-infra snippets) allows less code maintenance
and simlifies further development.

Rev. 0.1 Charts version bump for ceph-client, ceph-mon, ceph-osd,
ceph-provisioners and helm-toolkit
Rev. 0.2 Mon endpoint discovery functionality added for
the rados gateway. ClusterRole and ClusterRoleBinding added.
Rev. 0.3 checkdns is allowed to correct ceph.conf for RGW deployment.
Rev. 0.4 Added RoleBinding to the deployment-rgw.
Rev. 0.5 Remove _namespace-client-ceph-config-manager.sh.tpl and
         the appropriate job, because of duplicated functionality.
         Related configuration has been removed.
Rev. 0.6 RoleBinding logic has been changed to meet rules:
    checkdns namespace - HAS ACCESS -> RGW namespace(s)

Change-Id: Ie0af212bdcbbc3aa53335689deed9b226e5d4d89

ceph-client/Chart.yaml

@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph Client
 name: ceph-client
-version: 0.1.30
+version: 0.1.31
 home: https://github.com/ceph/ceph-client
 ...

ceph-client/templates/bin/_namespace-client-ceph-config-manager.sh.tpl

@@ -1,36 +0,0 @@
-#!/bin/bash
-
-{{/*
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-set -ex
-{{- $envAll := . }}
-
-{{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }}
-
-ENDPOINT=$(mon_host_from_k8s_ep "${DEPLOYMENT_NAMESPACE}" ceph-mon-discovery)
-
-if [[ -z "${ENDPOINT}" ]]; then
-  echo "Ceph Mon endpoint is empty"
-  exit 1
-else
-  echo "${ENDPOINT}"
-fi
-
-# Update the ceph-client-etc configmap
-kubectl get cm "${CEPH_CONF_ETC}" -n "${DEPLOYMENT_NAMESPACE}" -o json |
-  jq '.data."ceph.conf" |= sub("mon_host = .*";"mon_host = '"${ENDPOINT}"'")' |
-  kubectl apply -n "${DEPLOYMENT_NAMESPACE}" -f -
-
-kubectl get cm "${CEPH_CONF_ETC}" -n "${DEPLOYMENT_NAMESPACE}" -o yaml

ceph-client/templates/bin/mds/_start.sh.tpl

@@ -14,7 +14,7 @@ export LC_ALL=C
 : "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}"
 : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
 
-{{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }}
+{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
 
 if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"

ceph-client/templates/bin/pool/_init.sh.tpl

@@ -20,7 +20,7 @@ export LC_ALL=C
 : "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}"
 : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
 
-{{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }}
+{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
 
 if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"

ceph-client/templates/bin/utils/_checkDNS_start.sh.tpl

@@ -16,33 +16,50 @@ limitations under the License.
 
 set -xe
 
-{{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }}
+{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
+
+{{- $rgwNameSpaces := "" }}
+{{- $sep := "" }}
+{{- range $_, $ns := .Values.endpoints.ceph_object_store.endpoint_namespaces }}
+  {{- $rgwNameSpaces = printf "%s%s%s" $rgwNameSpaces $sep $ns }}
+  {{- $sep = " " }}
+{{- end }}
+
+rgwNameSpaces={{- printf "\"%s\"" $rgwNameSpaces }}
 
 function check_mon_dns {
-  DNS_CHECK=$(getent hosts ceph-mon | head -n1)
+  NS=${1}
-  PODS=$(kubectl get pods --namespace=${NAMESPACE} --selector=application=ceph --field-selector=status.phase=Running \
+  # RGWs and the rgw namespace could not exist. Let's check this and prevent this script from failing
-         --output=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E 'ceph-mon|ceph-osd|ceph-mgr|ceph-mds')
+  if [[ $(kubectl get ns ${NS} -o json | jq -r '.status.phase') == "Active" ]]; then
-  ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery)
+    DNS_CHECK=$(getent hosts ceph-mon | head -n1)
+    PODS=$(kubectl get pods --namespace=${NS} --selector=application=ceph --field-selector=status.phase=Running \
+          --output=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E 'ceph-mon|ceph-osd|ceph-mgr|ceph-mds|ceph-rgw')
+    ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery)
 
-  if [[ ${PODS} == "" || "${ENDPOINT}" == "" ]]; then
+    if [[ ${PODS} == "" || "${ENDPOINT}" == "" ]]; then
-    echo "Something went wrong, no PODS or ENDPOINTS are available!"
+      echo "Something went wrong, no PODS or ENDPOINTS are available!"
-  elif [[ ${DNS_CHECK} == "" ]]; then
+    elif [[ ${DNS_CHECK} == "" ]]; then
-    for POD in ${PODS}; do
+      for POD in ${PODS}; do
-      kubectl exec -t ${POD} --namespace=${NAMESPACE} -- \
+        kubectl exec -t ${POD} --namespace=${NS} -- \
-      sh -c -e "/tmp/utils-checkDNS.sh "${ENDPOINT}""
+        sh -c -e "/tmp/utils-checkDNS.sh "${ENDPOINT}""
-    done
+      done
+    else
+      for POD in ${PODS}; do
+        kubectl exec -t ${POD} --namespace=${NS} -- \
+        sh -c -e "/tmp/utils-checkDNS.sh up"
+      done
+    fi
   else
-    for POD in ${PODS}; do
+    echo "The namespace ${NS} is not ready, yet"
-      kubectl exec -t ${POD} --namespace=${NAMESPACE} -- \
-      sh -c -e "/tmp/utils-checkDNS.sh up"
-    done
   fi
 }
 
 function watch_mon_dns {
   while [ true ]; do
     echo "checking DNS health"
-    check_mon_dns || true
+    for myNS in ${NAMESPACE} ${rgwNameSpaces}; do
+      check_mon_dns ${myNS} || true
+    done
     echo "sleep 300 sec"
     sleep 300
   done

ceph-client/templates/configmap-bin.yaml

@@ -32,8 +32,6 @@ data:
 
   init-dirs.sh: |
 {{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  namespace-client-ceph-config-manager.sh: |
-{{ tuple "bin/_namespace-client-ceph-config-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 
   pool-init.sh: |
 {{ tuple "bin/pool/_init.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}

ceph-client/templates/deployment-checkdns.yaml

@@ -16,12 +16,19 @@ limitations under the License.
 {{- $envAll := . }}
 
 {{- $serviceAccountName := "ceph-checkdns" }}
+{{/*
+We will give different name to the RoleBinding resource (see $cephRoleBindingName variable below).
+This is neccessary, because the RoleBinding with the default name "ceph-checkdns" exists in the system,
+and its reference can not be changed.
+*/}}
+{{- $cephRoleBindingName := "ceph-checkdns-rolebinding" }}
+
 {{ tuple $envAll "checkdns" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
+kind: ClusterRole
 metadata:
-  name: {{ $serviceAccountName }}
+  name: clusterrole-checkdns
 rules:
   - apiGroups:
       - ""
@@ -29,25 +36,29 @@ rules:
       - pods
       - endpoints
       - pods/exec
+      - namespaces
     verbs:
       - get
       - list
       - watch
       - create
 ---
+
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: {{ $serviceAccountName }}
+  name: {{ printf "%s-for-%s" $cephRoleBindingName $envAll.Release.Namespace }}
+  namespace: {{ $envAll.Release.Namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: Role
+  kind: ClusterRole
-  name: {{ $serviceAccountName }}
+  name: clusterrole-checkdns
 subjects:
   - kind: ServiceAccount
     name: {{ $serviceAccountName }}
     namespace: {{ $envAll.Release.Namespace }}
 ---
+
 kind: Deployment
 apiVersion: apps/v1
 metadata:

ceph-client/templates/job-ns-client-ceph-config.yaml

@@ -1,114 +0,0 @@
-{{/*
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if and .Values.manifests.job_ns_client_ceph_config .Values.manifests.configmap_etc }}
-{{- $envAll := . }}
-
-{{- $serviceAccountName := "ceph-ns-client-ceph-config" }}
-{{ tuple $envAll "namespace_client_ceph_config_update" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: {{ $serviceAccountName }}
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-    verbs:
-      - get
-      - create
-      - update
-      - patch
-  - apiGroups:
-      - ""
-    resources:
-      - endpoints
-    verbs:
-      - get
-      - list
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: {{ $serviceAccountName }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: {{ $serviceAccountName }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ $serviceAccountName }}
-    namespace: {{ $envAll.Release.Namespace }}
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: {{ $serviceAccountName }}
-  labels:
-{{ tuple $envAll "ceph" "client-ceph-config-update" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
-spec:
-  template:
-    metadata:
-      labels:
-{{ tuple $envAll "ceph" "client-ceph-config-update" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
-      annotations:
-{{ dict "envAll" $envAll "podName" $serviceAccountName "containerNames" (list "ceph-client-config-update" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
-    spec:
-{{ dict "envAll" $envAll "application" "client_ceph_config_update" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
-      serviceAccountName: {{ $serviceAccountName }}
-      restartPolicy: {{ $envAll.Values.jobs.client_ceph_config_update.restartPolicy | quote }}
-      nodeSelector:
-        {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }}
-      initContainers:
-{{ tuple $envAll "namespace_client_ceph_config_update" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
-      containers:
-        - name: ceph-client-config-update
-{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }}
-{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-{{ dict "envAll" $envAll "application" "client_ceph_config_update" "container" "ceph_storage_keys_update" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
-          env:
-            - name: CEPH_CONF_ETC
-              value: "ceph-client-etc"
-            - name: DEPLOYMENT_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: MON_PORT
-              value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
-            - name: MON_PORT_V2
-              value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
-
-          command:
-            - /tmp/namespace-client-ceph-config-manager.sh
-          volumeMounts:
-            - name: pod-tmp
-              mountPath: /tmp
-            - name: pod-etc-ceph
-              mountPath: /etc/ceph
-            - name: ceph-client-bin-clients
-              mountPath: /tmp/namespace-client-ceph-config-manager.sh
-              subPath: namespace-client-ceph-config-manager.sh
-              readOnly: true
-      volumes:
-        - name: pod-tmp
-          emptyDir: {}
-        - name: pod-etc-ceph
-          emptyDir: {}
-        - name: ceph-client-bin-clients
-          configMap:
-            name: ceph-client-bin
-            defaultMode: 0555
-{{- end }}

ceph-client/values.yaml

@@ -220,8 +220,6 @@ jobs:
     startingDeadlineSecs: 60
   rbd_pool:
     restartPolicy: OnFailure
-  client_ceph_config_update:
-    restartPolicy: OnFailure
 
 conf:
   features:
@@ -540,6 +538,14 @@ endpoints:
         default: 9283
     scheme:
       default: http
+  ceph_object_store:
+    endpoint_namespaces:
+    - openstack
+    - ceph
+    # hosts:
+    #   default: ceph-rgw
+    # host_fqdn_override:
+    #   default: null
 
 manifests:
   configmap_bin: true
@@ -548,7 +554,6 @@ manifests:
   deployment_mds: true
   deployment_checkdns: true
   job_bootstrap: false
-  job_ns_client_ceph_config: true
   job_cephfs_client_key: true
   job_image_repo_sync: true
   job_rbd_pool: true

ceph-mon/Chart.yaml

@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph Mon
 name: ceph-mon
-version: 0.1.19
+version: 0.1.20
 home: https://github.com/ceph/ceph
 ...

ceph-mon/templates/bin/mgr/_start.sh.tpl

@@ -6,7 +6,7 @@ set -ex
 : "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}"
 : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
 
-{{ include "ceph-mon.snippets.mon_host_from_k8s_ep" . }}
+{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
 
 if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"

ceph-mon/templates/bin/mon/_start.sh.tpl

@@ -8,15 +8,16 @@ export LC_ALL=C
 : "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}"
 : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
 
+{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
+
 if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
   exit 1
 else
-  ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
+
-             -v version=v1 -v msgr_version=v2 \
+  ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery)
-             -v msgr2_port=${MON_PORT_V2} \
+
-             '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
+  if [[ -z "${ENDPOINT}" ]]; then
-  if [[ "${ENDPOINT}" == "" ]]; then
     /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
   else
     /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true

ceph-mon/templates/bin/moncheck/_start.sh.tpl

@@ -3,14 +3,13 @@ set -ex
 export LC_ALL=C
 : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
 
+{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
+
 if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
   exit 1
 else
-  ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
+  ENDPOINT=$(mon_host_from_k8s_ep ${NAMESPACE} ceph-mon-discovery)
-             -v version=v1 -v msgr_version=v2 \
-             -v msgr2_port=${MON_PORT_V2} \
-             '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
   if [[ "${ENDPOINT}" == "" ]]; then
     /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
   else

ceph-osd/Chart.yaml

@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph OSD
 name: ceph-osd
-version: 0.1.34
+version: 0.1.35
 home: https://github.com/ceph/ceph
 ...

ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl

@@ -28,6 +28,8 @@ export PS4='+${BASH_SOURCE:+$(basename ${BASH_SOURCE}):${LINENO}:}${FUNCNAME:+${
 : "${OSD_JOURNAL_SIZE:=$(awk '/^osd_journal_size/{print $3}' ${CEPH_CONF}.template)}"
 : "${OSD_WEIGHT:=1.0}"
 
+{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
+
 # Obtain a global lock on /var/lib/ceph/tmp/init-osd.lock
 function lock() {
   # Open a file descriptor for the lock file if there isn't one already
@@ -136,11 +138,8 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
   exit 1
 else
-  ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
+  ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery)
-             -v version=v1 -v msgr_version=v2 \
+  if [[ -z "${ENDPOINT}" ]]; then
-             -v msgr2_port=${MON_PORT_V2} \
-             '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
-  if [[ "${ENDPOINT}" == "" ]]; then
     /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
   else
     /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true

ceph-provisioners/Chart.yaml

@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph Provisioner
 name: ceph-provisioners
-version: 0.1.16
+version: 0.1.17
 home: https://github.com/ceph/ceph
 ...

ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-ceph-config-manager.sh.tpl

@@ -17,11 +17,9 @@ limitations under the License.
 set -ex
 {{- $envAll := . }}
 
+{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
 
-ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
+ENDPOINT=$(mon_host_from_k8s_ep ${PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE} ceph-mon-discovery)
-           -v version=v1 -v msgr_version=v2 \
-           -v msgr2_port=${MON_PORT_V2} \
-           '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
 
 if [ -z "$ENDPOINT" ]; then
   echo "Ceph Mon endpoint is empty"

ceph-rgw/Chart.yaml

@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph RadosGW
 name: ceph-rgw
-version: 0.1.17
+version: 0.1.19
 home: https://github.com/ceph/ceph
 ...

ceph-rgw/templates/bin/rgw/_init.sh.tpl

@@ -15,10 +15,32 @@ limitations under the License.
 */}}
 
 set -ex
+export LC_ALL=C
 
-cp -va /etc/ceph/ceph.conf.template /etc/ceph/ceph.conf
+: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
+: "${EP:=ceph-mon-discovery}"
+{{- if empty .Values.endpoints.ceph_mon.namespace -}}
+MON_NS=ceph
+{{ else }}
+MON_NS={{ .Values.endpoints.ceph_mon.namespace }}
+{{- end }}
 
-cat >> /etc/ceph/ceph.conf <<EOF
+{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
+
+if [[ ! -e ${CEPH_CONF}.template ]]; then
+  echo "ERROR- ${CEPH_CONF}.template must exist."
+  exit 1
+fi
+
+ENDPOINT=$(mon_host_from_k8s_ep "${MON_NS}" "${EP}")
+
+if [[ -z "${ENDPOINT}" ]]; then
+   /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
+else
+   /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true
+fi
+
+cat >> ${CEPH_CONF} <<EOF
 
 [client.rgw.$(hostname -s)]
 {{ range $key, $value := .Values.conf.rgw.config -}}

ceph-rgw/templates/bin/utils/_checkDNS.sh.tpl

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
+ENDPOINT="{$1}"
+
+function check_mon_dns () {
+  GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF})
+
+  if [[ "${ENDPOINT}" == "{up}" ]]; then
+    echo "If DNS is working, we are good here"
+  elif [[ "${ENDPOINT}" != "" ]]; then
+    if [[ ${GREP_CMD} != "" ]]; then
+      # No DNS, write CEPH MONs IPs into ${CEPH_CONF}
+      sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1
+    else
+      echo "endpoints are already cached in ${CEPH_CONF}"
+      exit
+    fi
+  fi
+}
+
+check_mon_dns
+
+exit

ceph-rgw/templates/configmap-bin.yaml

@@ -29,7 +29,6 @@ data:
   bootstrap.sh: |
 {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
-
   rgw-restart.sh: |
 {{ tuple "bin/_rgw-restart.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   init-dirs.sh: |
@@ -49,4 +48,6 @@ data:
 {{ tuple "bin/_create-rgw-placement-targets.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   helm-tests.sh: |
 {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  utils-checkDNS.sh: |
+{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}

ceph-rgw/templates/deployment-rgw.yaml

@@ -23,6 +23,7 @@ limitations under the License.
 {{- end }}
 
 {{- $serviceAccountName := "ceph-rgw" }}
+{{- $checkDnsServiceAccountName := "ceph-checkdns" }}
 
 {{- $_ := set $envAll.Values "__depParams" ( list ) }}
 {{- if .Values.conf.rgw_ks.enabled -}}
@@ -37,6 +38,52 @@ limitations under the License.
 {{- $_ := include "helm-toolkit.utils.dependency_resolver" $dependencyOpts | toString | fromYaml }}
 {{ tuple $envAll "pod_dependency" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+  namespace: {{ .Values.endpoints.ceph_mon.namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+  namespace: {{ .Values.endpoints.ceph_mon.namespace }}
+roleRef:
+  kind: Role
+  name: {{ $serviceAccountName }}
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+# This role bindig refers to the ClusterRole for
+# check-dns deployment.
+# See: openstack-helm-infra/ceph-client/deployment-checkdns.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ printf "%s-from-%s-to-%s" $checkDnsServiceAccountName $envAll.Values.endpoints.ceph_mon.namespace $envAll.Release.Namespace }}
+  namespace: {{ $envAll.Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: clusterrole-checkdns
+subjects:
+  - kind: ServiceAccount
+    name: {{ $checkDnsServiceAccountName }}
+    namespace:  {{ .Values.endpoints.ceph_mon.namespace }}
+---
 kind: Deployment
 apiVersion: apps/v1
 metadata:
@@ -172,6 +219,10 @@ spec:
               mountPath: /tmp/rgw-start.sh
               subPath: rgw-start.sh
               readOnly: true
+            - name: ceph-rgw-bin
+              mountPath: /tmp/utils-checkDNS.sh
+              subPath: utils-checkDNS.sh
+              readOnly: true
             - name: ceph-rgw-etc
               mountPath: /etc/ceph/ceph.conf.template
               subPath: ceph.conf

helm-toolkit/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Helm-Toolkit
 name: helm-toolkit
-version: 0.2.31
+version: 0.2.32
 home: https://docs.openstack.org/openstack-helm
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
 sources:

helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_namespace_lookup.tpl

@@ -0,0 +1,38 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Resolves the namespace scoped hostname for an endpoint
+values: |
+  endpoints:
+    oslo_db:
+      hosts:
+        default: mariadb
+      host_fqdn_override:
+        default: null
+usage: |
+  {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_namespace_lookup" }}
+return: |
+  default
+*/}}
+
+{{- define "helm-toolkit.endpoints.hostname_namespaced_endpoint_namespace_lookup" -}}
+{{- $type := index . 0 -}}
+{{- $endpoint := index . 1 -}}
+{{- $context := index . 2 -}}
+{{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }}
+{{- $namespace := $endpointMap.namespace | default $context.Release.Namespace }}
+{{- printf "%s" $namespace -}}
+{{- end -}}

helm-toolkit/templates/snippets/_mon_host_from_k8s_ep.sh.tpl

@@ -1,4 +1,4 @@
-{{- define "ceph-client.snippets.mon_host_from_k8s_ep" -}}
+{{- define "helm-toolkit.snippets.mon_host_from_k8s_ep" -}}
 {{/*
 
 Inserts a bash function definition mon_host_from_k8s_ep() which can be used
@@ -10,7 +10,7 @@ Usage (e.g. in _script.sh.tpl):
     : "${NS:=ceph}"
     : "${EP:=ceph-mon-discovery}"
 
-    {{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }}
+    {{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
 
     MON_HOST=$(mon_host_from_k8s_ep "$NS" "$EP")
 

releasenotes/notes/ceph-client.yaml

@@ -31,4 +31,5 @@ ceph-client:
   - 0.1.28 Fix ceph.conf update job labels, rendering
   - 0.1.29 Consolidate mon_host discovery
   - 0.1.30 Move ceph-mgr deployment to the ceph-mon chart
+  - 0.1.31 Consolidate mon_endpoints discovery
 ...

releasenotes/notes/ceph-mon.yaml

@@ -20,4 +20,5 @@ ceph-mon:
   - 0.1.17 Skip monmap endpoint check for missing mons
   - 0.1.18 Move ceph-mgr deployment to the ceph-mon chart
   - 0.1.19 Add a post-apply job to restart mons after mgrs
+  - 0.1.20 Consolidate mon_endpoints discovery
 ...

releasenotes/notes/ceph-osd.yaml

@@ -35,4 +35,5 @@ ceph-osd:
   - 0.1.32 Update htk requirements
   - 0.1.33 Update log-runner container for MAC
   - 0.1.34 Remove wait for misplaced objects during OSD restarts
+  - 0.1.35 Consolidate mon_endpoints discovery
 ...

releasenotes/notes/ceph-provisioners.yaml

@@ -16,4 +16,5 @@ ceph-provisioners:
   - 0.1.14 Helm 3 - Fix Job labels
   - 0.1.15 Add support to connect to rook-ceph cluster
   - 0.1.16 Update htk requirements
+  - 0.1.17 Consolidate mon_endpoints discovery
 ...

releasenotes/notes/ceph-rgw.yaml

@@ -18,4 +18,6 @@ ceph-rgw:
   - 0.1.15 Correct rgw placement target functions
   - 0.1.16 Helm 3 - Fix Job labels
   - 0.1.17 Update htk requirements
+  - 0.1.18 Consolidate mon_endpoints discovery
+  - 0.1.19 Add ClusterRole to the bootstrap-job
 ...

releasenotes/notes/helm-toolkit.yaml

@@ -38,4 +38,5 @@ helm-toolkit:
   - 0.2.29 Enhance mariadb backup
   - 0.2.30 Add ability to image pull secrets on pods
   - 0.2.31 Add log strings for alert generation
+  - 0.2.32 Consolidate mon_endpoints discovery
 ...