diff --git a/ceph-client/templates/configmap-etc-client.yaml b/ceph-client/templates/configmap-etc-client.yaml index 0e1610b927..fb2a5adc8e 100644 --- a/ceph-client/templates/configmap-etc-client.yaml +++ b/ceph-client/templates/configmap-etc-client.yaml @@ -22,10 +22,15 @@ limitations under the License. {{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }} {{- if empty .Values.conf.ceph.global.mon_host -}} -{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} {{- end -}} +{{- if empty .Values.conf.ceph.global.mon_addr -}} +{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}} +{{- end -}} + {{- if empty .Values.conf.ceph.osd.cluster_network -}} {{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}} {{- end -}} diff --git a/ceph-mon/templates/bin/mon/_start.sh.tpl b/ceph-mon/templates/bin/mon/_start.sh.tpl index 24b6ed8e21..9588d9384b 100644 --- a/ceph-mon/templates/bin/mon/_start.sh.tpl +++ b/ceph-mon/templates/bin/mon/_start.sh.tpl @@ -39,9 +39,9 @@ function get_mon_config { while [[ -z "${MONMAP_ADD// }" && "${timeout}" -gt 0 ]]; do # Get the ceph mon pods (name and IP) from the Kubernetes API. Formatted as a set of monmap params if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then - MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}} {{`{{end}}`}} {{`{{end}}`}}") + MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}") else - MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}} {{`{{end}}`}} {{`{{end}}`}}") + MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}") fi (( timeout-- )) sleep 1 @@ -53,7 +53,7 @@ function get_mon_config { # if monmap exists and the mon is already there, don't overwrite monmap if [ -f "${MONMAP}" ]; then - monmaptool --print "${MONMAP}" |grep -q "${MON_IP// }"":6789" + monmaptool --print "${MONMAP}" |grep -q "${MON_IP// }"":${MON_PORT}" if [ $? -eq 0 ]; then echo "${MON_IP} already exists in monmap ${MONMAP}" return @@ -93,7 +93,7 @@ else # no mons are up and running yet timeout 5 ceph --cluster "${CLUSTER}" mon getmap -o ${MONMAP} || true ceph-mon --setuser ceph --setgroup ceph --cluster "${CLUSTER}" -i ${MON_NAME} --inject-monmap ${MONMAP} --keyring ${MON_KEYRING} --mon-data "${MON_DATA_DIR}" - timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:6789" || true + timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:${MON_PORT}" || true fi # start MON @@ -104,4 +104,4 @@ exec /usr/bin/ceph-mon \ -d \ -i ${MON_NAME} \ --mon-data "${MON_DATA_DIR}" \ - --public-addr "${MON_IP}:6789" + --public-addr "${MON_IP}:${MON_PORT}" diff --git a/ceph-mon/templates/configmap-etc.yaml b/ceph-mon/templates/configmap-etc.yaml index 148dcc2249..ac4b1e7b03 100644 --- a/ceph-mon/templates/configmap-etc.yaml +++ b/ceph-mon/templates/configmap-etc.yaml @@ -22,10 +22,15 @@ limitations under the License. {{- if .Values.deployment.ceph }} {{- if empty .Values.conf.ceph.global.mon_host -}} -{{- $monHost := tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $monHost := tuple "ceph_mon" "discovery" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} {{- end -}} +{{- if empty .Values.conf.ceph.global.mon_addr -}} +{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}} +{{- end -}} + {{- if empty .Values.conf.ceph.global.fsid -}} {{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}} {{- end -}} diff --git a/ceph-mon/templates/daemonset-mon.yaml b/ceph-mon/templates/daemonset-mon.yaml index c70435b294..4709436309 100644 --- a/ceph-mon/templates/daemonset-mon.yaml +++ b/ceph-mon/templates/daemonset-mon.yaml @@ -106,6 +106,8 @@ spec: value: {{ .Values.network.public | quote }} - name: KUBECTL_PARAM value: {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_kubectl_params" | indent 10 }} + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: MON_IP valueFrom: fieldRef: @@ -126,7 +128,7 @@ spec: command: - /tmp/mon-stop.sh ports: - - containerPort: 6789 + - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} livenessProbe: exec: command: diff --git a/ceph-mon/templates/deployment-moncheck.yaml b/ceph-mon/templates/deployment-moncheck.yaml index ffce5c52d6..70f8e109ae 100644 --- a/ceph-mon/templates/deployment-moncheck.yaml +++ b/ceph-mon/templates/deployment-moncheck.yaml @@ -60,7 +60,7 @@ spec: command: - /tmp/moncheck-start.sh ports: - - containerPort: 6789 + - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} volumeMounts: - name: ceph-mon-bin mountPath: /tmp/moncheck-start.sh diff --git a/ceph-osd/templates/configmap-etc.yaml b/ceph-osd/templates/configmap-etc.yaml index e37eeb0765..85ab873b9e 100644 --- a/ceph-osd/templates/configmap-etc.yaml +++ b/ceph-osd/templates/configmap-etc.yaml @@ -20,10 +20,15 @@ limitations under the License. {{- with $envAll }} {{- if empty .Values.conf.ceph.global.mon_host -}} -{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} {{- end -}} +{{- if empty .Values.conf.ceph.global.mon_addr -}} +{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}} +{{- end -}} + {{- if empty .Values.conf.ceph.global.fsid -}} {{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}} {{- end -}} diff --git a/ceph-osd/values.yaml b/ceph-osd/values.yaml index 7ee506b026..c40dd43dc1 100644 --- a/ceph-osd/values.yaml +++ b/ceph-osd/values.yaml @@ -79,10 +79,6 @@ secrets: network: public: 192.168.0.0/16 cluster: 192.168.0.0/16 - port: - mon: 6789 - rgw: 8088 - mgr: 7000 conf: ceph: diff --git a/ceph-provisioners/templates/configmap-etc-client.yaml b/ceph-provisioners/templates/configmap-etc-client.yaml index 3592990580..e3001bd686 100644 --- a/ceph-provisioners/templates/configmap-etc-client.yaml +++ b/ceph-provisioners/templates/configmap-etc-client.yaml @@ -22,10 +22,15 @@ limitations under the License. {{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }} {{- if empty .Values.conf.ceph.global.mon_host -}} -{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} {{- end -}} +{{- if empty .Values.conf.ceph.global.mon_addr -}} +{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}} +{{- end -}} + {{- if empty .Values.conf.ceph.osd.cluster_network -}} {{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}} {{- end -}} diff --git a/ceph-provisioners/templates/deployment-cephfs-provisioner.yaml b/ceph-provisioners/templates/deployment-cephfs-provisioner.yaml index 6548a31738..4830fba4e9 100644 --- a/ceph-provisioners/templates/deployment-cephfs-provisioner.yaml +++ b/ceph-provisioners/templates/deployment-cephfs-provisioner.yaml @@ -91,7 +91,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: - name: run-cephfs-provisioner + name: {{ $serviceAccountName }}-run-cephfs-provisioner subjects: - kind: ServiceAccount name: {{ $serviceAccountName }} diff --git a/ceph-provisioners/templates/deployment-rbd-provisioner.yaml b/ceph-provisioners/templates/deployment-rbd-provisioner.yaml index ad21d36af0..99c7e3016b 100644 --- a/ceph-provisioners/templates/deployment-rbd-provisioner.yaml +++ b/ceph-provisioners/templates/deployment-rbd-provisioner.yaml @@ -81,7 +81,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: - name: run-rbd-provisioner + name: {{ $serviceAccountName }}-run-rbd-provisioner subjects: - kind: ServiceAccount name: {{ $serviceAccountName }} diff --git a/ceph-provisioners/values.yaml b/ceph-provisioners/values.yaml index 2b8fd166e7..feb817e7d0 100644 --- a/ceph-provisioners/values.yaml +++ b/ceph-provisioners/values.yaml @@ -98,9 +98,6 @@ secrets: network: public: 192.168.0.0/16 cluster: 192.168.0.0/16 - port: - mon: 6789 - mgr: 7000 conf: ceph: diff --git a/tools/deployment/developer/ceph/040-ceph.sh b/tools/deployment/developer/ceph/040-ceph.sh index f389d06c62..53ba741b74 100755 --- a/tools/deployment/developer/ceph/040-ceph.sh +++ b/tools/deployment/developer/ceph/040-ceph.sh @@ -40,11 +40,29 @@ endpoints: namespace: openstack object_store: namespace: ceph + port: + api: + default: 8088 + public: 80 ceph_mon: namespace: ceph + port: + mon: + default: 6789 + ceph_mgr: + namespace: ceph + port: + mgr: + default: 7000 + metrics: + default: 9283 network: public: 172.17.0.1/16 cluster: 172.17.0.1/16 + port: + mon: 6789 + rgw: 8088 + mgr: 7000 deployment: storage_secrets: true ceph: true @@ -60,6 +78,7 @@ conf: ceph: global: fsid: ${CEPH_FS_ID} + mon_addr: :6789 osd_pool_default_size: 1 osd: osd_crush_chooseleaf_type: 0