[ceph-mon] Add mon hosts as per msgr2 protocol

This is to update mon_host configuration to support both v1 and v2
of messenger.

ex: mon_host = [v1:172.29.0.11:6790/0,v2:172.29.0.11:3300/0]

Change-Id: I02785ea42c07d1aecbef2cf0c32dd6a1a236659f
Signed-off-by: Pete Birley <pete@port.direct>

ceph-mon/templates/bin/mon/_start.sh.tpl

@@ -12,11 +12,14 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
   exit 1
 else
-  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/"ip"/{print $4":"port}' | paste -sd',')
+  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
-  if [[ ${ENDPOINT} == "" ]]; then
+             -v version=v1 -v msgr_version=v2 \
+             -v msgr2_port=${MON_PORT_V2} \
+             '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
+  if [[ "${ENDPOINT}" == "" ]]; then
     /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
   else
-    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true
   fi
 fi
 

ceph-mon/templates/configmap-etc.yaml

@@ -26,11 +26,6 @@ limitations under the License.
 {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
 {{- end -}}
 
-{{- if empty .Values.conf.ceph.global.mon_addr -}}
-{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
-{{- end -}}
-
 {{- if empty .Values.conf.ceph.global.fsid -}}
 {{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}}
 {{- end -}}

ceph-mon/templates/daemonset-mon.yaml

@@ -141,6 +141,8 @@ spec:
               value: {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_kubectl_params" | indent 10 }}
             - name: MON_PORT
               value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: MON_PORT_V2
+              value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             - name: MON_IP
               valueFrom:
                 fieldRef:
@@ -162,6 +164,7 @@ spec:
                     - /tmp/mon-stop.sh
           ports:
             - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - containerPort: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           livenessProbe:
            exec:
             command:

ceph-mon/templates/service-mon-discovery.yaml

@@ -31,9 +31,14 @@ metadata:
     service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
 spec:
   ports:
-  - port: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+  - name: mon
+    port: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
     protocol: TCP
     targetPort: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+  - name: mon-msgr2
+    port: {{ tuple "ceph_mon" "discovery" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    protocol: TCP
+    targetPort: {{ tuple "ceph_mon" "discovery" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
   selector:
 {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
   clusterIP: None

ceph-mon/templates/service-mon.yaml

@@ -23,9 +23,14 @@ metadata:
   name: {{ tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 spec:
   ports:
-  - port: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+  - name: mon
+    port: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
     protocol: TCP
     targetPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+  - name: mon-msgr2
+    port: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    protocol: TCP
+    targetPort: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
   selector:
 {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
   clusterIP: None

ceph-mon/values.yaml

@@ -46,7 +46,7 @@ labels:
 
 pod:
   mandatory_access_control:
-    type: apparmor
+    type: null
     ceph-mon:
       ceph-mon: runtime/default
   security_context:
@@ -328,6 +328,8 @@ endpoints:
     port:
       mon:
         default: 6789
+      mon_msgr2:
+        default: 3300
 
 manifests:
   configmap_bin: true