Make Ceph Mon port configurable

- make mon port overridable through env variable
- update ceph.conf to include fqdn with port number
- add mon_add: to ceph global conf
- updating ClusterRoleBinding name for cephfs and rbd
 provisioner.
- remove unused network.port section as we specify ports
 through endpoints

With these changes, ceph mon can run on different port

Change-Id: I8af2bbe3ee84fd3692bdff92d821b9621a6d53c9
This commit is contained in:
Renis 2018-08-08 21:22:52 -07:00
parent fbfcb51c31
commit f3b7c7fc82
12 changed files with 54 additions and 20 deletions

View File

@ -22,10 +22,15 @@ limitations under the License.
{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }} {{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
{{- if empty .Values.conf.ceph.global.mon_host -}} {{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} {{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}} {{- end -}}
{{- if empty .Values.conf.ceph.global.mon_addr -}}
{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.osd.cluster_network -}} {{- if empty .Values.conf.ceph.osd.cluster_network -}}
{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}} {{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}
{{- end -}} {{- end -}}

View File

@ -39,9 +39,9 @@ function get_mon_config {
while [[ -z "${MONMAP_ADD// }" && "${timeout}" -gt 0 ]]; do while [[ -z "${MONMAP_ADD// }" && "${timeout}" -gt 0 ]]; do
# Get the ceph mon pods (name and IP) from the Kubernetes API. Formatted as a set of monmap params # Get the ceph mon pods (name and IP) from the Kubernetes API. Formatted as a set of monmap params
if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then
MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}} {{`{{end}}`}} {{`{{end}}`}}") MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}")
else else
MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}} {{`{{end}}`}} {{`{{end}}`}}") MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}")
fi fi
(( timeout-- )) (( timeout-- ))
sleep 1 sleep 1
@ -53,7 +53,7 @@ function get_mon_config {
# if monmap exists and the mon is already there, don't overwrite monmap # if monmap exists and the mon is already there, don't overwrite monmap
if [ -f "${MONMAP}" ]; then if [ -f "${MONMAP}" ]; then
monmaptool --print "${MONMAP}" |grep -q "${MON_IP// }"":6789" monmaptool --print "${MONMAP}" |grep -q "${MON_IP// }"":${MON_PORT}"
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
echo "${MON_IP} already exists in monmap ${MONMAP}" echo "${MON_IP} already exists in monmap ${MONMAP}"
return return
@ -93,7 +93,7 @@ else
# no mons are up and running yet # no mons are up and running yet
timeout 5 ceph --cluster "${CLUSTER}" mon getmap -o ${MONMAP} || true timeout 5 ceph --cluster "${CLUSTER}" mon getmap -o ${MONMAP} || true
ceph-mon --setuser ceph --setgroup ceph --cluster "${CLUSTER}" -i ${MON_NAME} --inject-monmap ${MONMAP} --keyring ${MON_KEYRING} --mon-data "${MON_DATA_DIR}" ceph-mon --setuser ceph --setgroup ceph --cluster "${CLUSTER}" -i ${MON_NAME} --inject-monmap ${MONMAP} --keyring ${MON_KEYRING} --mon-data "${MON_DATA_DIR}"
timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:6789" || true timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:${MON_PORT}" || true
fi fi
# start MON # start MON
@ -104,4 +104,4 @@ exec /usr/bin/ceph-mon \
-d \ -d \
-i ${MON_NAME} \ -i ${MON_NAME} \
--mon-data "${MON_DATA_DIR}" \ --mon-data "${MON_DATA_DIR}" \
--public-addr "${MON_IP}:6789" --public-addr "${MON_IP}:${MON_PORT}"

View File

@ -22,10 +22,15 @@ limitations under the License.
{{- if .Values.deployment.ceph }} {{- if .Values.deployment.ceph }}
{{- if empty .Values.conf.ceph.global.mon_host -}} {{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} {{- $monHost := tuple "ceph_mon" "discovery" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}} {{- end -}}
{{- if empty .Values.conf.ceph.global.mon_addr -}}
{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.fsid -}} {{- if empty .Values.conf.ceph.global.fsid -}}
{{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}} {{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}}
{{- end -}} {{- end -}}

View File

@ -106,6 +106,8 @@ spec:
value: {{ .Values.network.public | quote }} value: {{ .Values.network.public | quote }}
- name: KUBECTL_PARAM - name: KUBECTL_PARAM
value: {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_kubectl_params" | indent 10 }} value: {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_kubectl_params" | indent 10 }}
- name: MON_PORT
value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: MON_IP - name: MON_IP
valueFrom: valueFrom:
fieldRef: fieldRef:
@ -126,7 +128,7 @@ spec:
command: command:
- /tmp/mon-stop.sh - /tmp/mon-stop.sh
ports: ports:
- containerPort: 6789 - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
livenessProbe: livenessProbe:
exec: exec:
command: command:

View File

@ -60,7 +60,7 @@ spec:
command: command:
- /tmp/moncheck-start.sh - /tmp/moncheck-start.sh
ports: ports:
- containerPort: 6789 - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
volumeMounts: volumeMounts:
- name: ceph-mon-bin - name: ceph-mon-bin
mountPath: /tmp/moncheck-start.sh mountPath: /tmp/moncheck-start.sh

View File

@ -20,10 +20,15 @@ limitations under the License.
{{- with $envAll }} {{- with $envAll }}
{{- if empty .Values.conf.ceph.global.mon_host -}} {{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} {{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}} {{- end -}}
{{- if empty .Values.conf.ceph.global.mon_addr -}}
{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.fsid -}} {{- if empty .Values.conf.ceph.global.fsid -}}
{{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}} {{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}}
{{- end -}} {{- end -}}

View File

@ -79,10 +79,6 @@ secrets:
network: network:
public: 192.168.0.0/16 public: 192.168.0.0/16
cluster: 192.168.0.0/16 cluster: 192.168.0.0/16
port:
mon: 6789
rgw: 8088
mgr: 7000
conf: conf:
ceph: ceph:

View File

@ -22,10 +22,15 @@ limitations under the License.
{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }} {{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
{{- if empty .Values.conf.ceph.global.mon_host -}} {{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} {{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}} {{- end -}}
{{- if empty .Values.conf.ceph.global.mon_addr -}}
{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.osd.cluster_network -}} {{- if empty .Values.conf.ceph.osd.cluster_network -}}
{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}} {{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}
{{- end -}} {{- end -}}

View File

@ -91,7 +91,7 @@ rules:
apiVersion: rbac.authorization.k8s.io/v1beta1 apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding kind: ClusterRoleBinding
metadata: metadata:
name: run-cephfs-provisioner name: {{ $serviceAccountName }}-run-cephfs-provisioner
subjects: subjects:
- kind: ServiceAccount - kind: ServiceAccount
name: {{ $serviceAccountName }} name: {{ $serviceAccountName }}

View File

@ -81,7 +81,7 @@ rules:
apiVersion: rbac.authorization.k8s.io/v1beta1 apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding kind: ClusterRoleBinding
metadata: metadata:
name: run-rbd-provisioner name: {{ $serviceAccountName }}-run-rbd-provisioner
subjects: subjects:
- kind: ServiceAccount - kind: ServiceAccount
name: {{ $serviceAccountName }} name: {{ $serviceAccountName }}

View File

@ -98,9 +98,6 @@ secrets:
network: network:
public: 192.168.0.0/16 public: 192.168.0.0/16
cluster: 192.168.0.0/16 cluster: 192.168.0.0/16
port:
mon: 6789
mgr: 7000
conf: conf:
ceph: ceph:

View File

@ -40,11 +40,29 @@ endpoints:
namespace: openstack namespace: openstack
object_store: object_store:
namespace: ceph namespace: ceph
port:
api:
default: 8088
public: 80
ceph_mon: ceph_mon:
namespace: ceph namespace: ceph
port:
mon:
default: 6789
ceph_mgr:
namespace: ceph
port:
mgr:
default: 7000
metrics:
default: 9283
network: network:
public: 172.17.0.1/16 public: 172.17.0.1/16
cluster: 172.17.0.1/16 cluster: 172.17.0.1/16
port:
mon: 6789
rgw: 8088
mgr: 7000
deployment: deployment:
storage_secrets: true storage_secrets: true
ceph: true ceph: true
@ -60,6 +78,7 @@ conf:
ceph: ceph:
global: global:
fsid: ${CEPH_FS_ID} fsid: ${CEPH_FS_ID}
mon_addr: :6789
osd_pool_default_size: 1 osd_pool_default_size: 1
osd: osd:
osd_crush_chooseleaf_type: 0 osd_crush_chooseleaf_type: 0