Make Ceph Mon port configurable

- make mon port overridable through env variable
- update ceph.conf to include fqdn with port number
- add mon_add: to ceph global conf
- updating ClusterRoleBinding name for cephfs and rbd
 provisioner.
- remove unused network.port section as we specify ports
 through endpoints

With these changes, ceph mon can run on different port

Change-Id: I8af2bbe3ee84fd3692bdff92d821b9621a6d53c9

ceph-client/templates/configmap-etc-client.yaml

@@ -22,10 +22,15 @@ limitations under the License.
 {{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
 
 {{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
 {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
 {{- end -}}
 
+{{- if empty .Values.conf.ceph.global.mon_addr -}}
+{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
+{{- end -}}
+
 {{- if empty .Values.conf.ceph.osd.cluster_network -}}
 {{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}
 {{- end -}}

ceph-mon/templates/bin/mon/_start.sh.tpl

@@ -39,9 +39,9 @@ function get_mon_config {
   while [[ -z "${MONMAP_ADD// }" && "${timeout}" -gt 0 ]]; do
     # Get the ceph mon pods (name and IP) from the Kubernetes API. Formatted as a set of monmap params
     if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then
-        MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}} {{`{{end}}`}} {{`{{end}}`}}")
+        MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}")
     else
-        MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}} {{`{{end}}`}} {{`{{end}}`}}")
+        MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}")
     fi
     (( timeout-- ))
     sleep 1
@@ -53,7 +53,7 @@ function get_mon_config {
 
   # if monmap exists and the mon is already there, don't overwrite monmap
   if [ -f "${MONMAP}" ]; then
-      monmaptool --print "${MONMAP}" |grep -q "${MON_IP// }"":6789"
+      monmaptool --print "${MONMAP}" |grep -q "${MON_IP// }"":${MON_PORT}"
       if [ $? -eq 0 ]; then
           echo "${MON_IP} already exists in monmap ${MONMAP}"
           return
@@ -93,7 +93,7 @@ else
   # no mons are up and running yet
   timeout 5 ceph --cluster "${CLUSTER}" mon getmap -o ${MONMAP} || true
   ceph-mon --setuser ceph --setgroup ceph --cluster "${CLUSTER}" -i ${MON_NAME} --inject-monmap ${MONMAP} --keyring ${MON_KEYRING} --mon-data "${MON_DATA_DIR}"
-  timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:6789" || true
+  timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:${MON_PORT}" || true
 fi
 
 # start MON
@@ -104,4 +104,4 @@ exec /usr/bin/ceph-mon \
   -d \
   -i ${MON_NAME} \
   --mon-data "${MON_DATA_DIR}" \
-  --public-addr "${MON_IP}:6789"
+  --public-addr "${MON_IP}:${MON_PORT}"

ceph-mon/templates/configmap-etc.yaml

@@ -22,10 +22,15 @@ limitations under the License.
 {{- if .Values.deployment.ceph }}
 
 {{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $monHost := tuple "ceph_mon" "discovery" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
 {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
 {{- end -}}
 
+{{- if empty .Values.conf.ceph.global.mon_addr -}}
+{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
+{{- end -}}
+
 {{- if empty .Values.conf.ceph.global.fsid -}}
 {{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}}
 {{- end -}}

ceph-mon/templates/daemonset-mon.yaml

@@ -106,6 +106,8 @@ spec:
               value: {{ .Values.network.public | quote }}
             - name: KUBECTL_PARAM
               value: {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_kubectl_params" | indent 10 }}
+            - name: MON_PORT
+              value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             - name: MON_IP
               valueFrom:
                 fieldRef:
@@ -126,7 +128,7 @@ spec:
                   command:
                     - /tmp/mon-stop.sh
           ports:
-            - containerPort: 6789
+            - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           livenessProbe:
            exec:
             command:

ceph-mon/templates/deployment-moncheck.yaml

@@ -60,7 +60,7 @@ spec:
           command:
             - /tmp/moncheck-start.sh
           ports:
-            - containerPort: 6789
+            - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           volumeMounts:
             - name: ceph-mon-bin
               mountPath: /tmp/moncheck-start.sh

ceph-osd/templates/configmap-etc.yaml

@@ -20,10 +20,15 @@ limitations under the License.
 {{- with $envAll }}
 
 {{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
 {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
 {{- end -}}
 
+{{- if empty .Values.conf.ceph.global.mon_addr -}}
+{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
+{{- end -}}
+
 {{- if empty .Values.conf.ceph.global.fsid -}}
 {{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}}
 {{- end -}}

ceph-osd/values.yaml

@@ -79,10 +79,6 @@ secrets:
 network:
   public: 192.168.0.0/16
   cluster: 192.168.0.0/16
-  port:
-    mon: 6789
-    rgw: 8088
-    mgr: 7000
 
 conf:
   ceph:

ceph-provisioners/templates/configmap-etc-client.yaml

@@ -22,10 +22,15 @@ limitations under the License.
 {{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
 
 {{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
 {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
 {{- end -}}
 
+{{- if empty .Values.conf.ceph.global.mon_addr -}}
+{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
+{{- end -}}
+
 {{- if empty .Values.conf.ceph.osd.cluster_network -}}
 {{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}
 {{- end -}}

ceph-provisioners/templates/deployment-cephfs-provisioner.yaml

@@ -91,7 +91,7 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
 metadata:
-  name: run-cephfs-provisioner
+  name: {{ $serviceAccountName }}-run-cephfs-provisioner
 subjects:
   - kind: ServiceAccount
     name: {{ $serviceAccountName }}

ceph-provisioners/templates/deployment-rbd-provisioner.yaml

@@ -81,7 +81,7 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
 metadata:
-  name: run-rbd-provisioner
+  name: {{ $serviceAccountName }}-run-rbd-provisioner
 subjects:
   - kind: ServiceAccount
     name: {{ $serviceAccountName }}

ceph-provisioners/values.yaml

@@ -98,9 +98,6 @@ secrets:
 network:
   public: 192.168.0.0/16
   cluster: 192.168.0.0/16
-  port:
-    mon: 6789
-    mgr: 7000
 
 conf:
   ceph:

tools/deployment/developer/ceph/040-ceph.sh

@@ -40,11 +40,29 @@ endpoints:
     namespace: openstack
   object_store:
     namespace: ceph
+    port:
+      api:
+        default: 8088
+        public: 80
   ceph_mon:
     namespace: ceph
+    port:
+      mon:
+        default: 6789
+  ceph_mgr:
+    namespace: ceph
+    port:
+      mgr:
+        default: 7000
+      metrics:
+        default: 9283
 network:
   public: 172.17.0.1/16
   cluster: 172.17.0.1/16
+  port:
+    mon: 6789
+    rgw: 8088
+    mgr: 7000
 deployment:
   storage_secrets: true
   ceph: true
@@ -60,6 +78,7 @@ conf:
   ceph:
     global:
       fsid: ${CEPH_FS_ID}
+      mon_addr: :6789
       osd_pool_default_size: 1
     osd:
       osd_crush_chooseleaf_type: 0