From 054affa29078ce5f5415b571300af7bbe0577995 Mon Sep 17 00:00:00 2001 From: Chinasubbareddy Mallavarapu Date: Tue, 1 Sep 2020 14:49:19 -0500 Subject: [PATCH] [Cinder] Support for external ceph backend This is to add support for external ceph cluster as second backend for cinder. prerequisite: - create ceph.conf for external ceph cluster as configmap - create cinder2 user and keyring on external ceph cluster - create keyring of user cinder2 as a secret now point configmap and secret created as prerequisite under values.ceph_client.external_ceph section also we need to add second backend under values.conf.backends section with all the details of external ceph cluster. Change-Id: I8df9f1da7208304f479dbb70b19af97fb01cd38f --- cinder/templates/bin/_ceph-keyring.sh.tpl | 10 ++++++++ cinder/templates/deployment-volume.yaml | 29 +++++++++++++++++++++++ cinder/templates/job-storage-init.yaml | 4 ++++ cinder/values.yaml | 9 +++++++ 4 files changed, 52 insertions(+) diff --git a/cinder/templates/bin/_ceph-keyring.sh.tpl b/cinder/templates/bin/_ceph-keyring.sh.tpl index de1cfcd9fe..e1bd624bca 100644 --- a/cinder/templates/bin/_ceph-keyring.sh.tpl +++ b/cinder/templates/bin/_ceph-keyring.sh.tpl @@ -22,4 +22,14 @@ cat < /etc/ceph/ceph.client.${RBD_USER}.keyring key = $(cat /tmp/client-keyring) EOF +if [ -f "/tmp/external-ceph-client-keyring" ]; then + RBD_USER=cinder2 + cat < /etc/ceph/ceph.client.${RBD_USER}.keyring +[client.${RBD_USER}] + key = $(cat /tmp/external-ceph-client-keyring) +EOF2 + +fi + + exit 0 diff --git a/cinder/templates/deployment-volume.yaml b/cinder/templates/deployment-volume.yaml index 776a49b247..8e96e0390c 100755 --- a/cinder/templates/deployment-volume.yaml +++ b/cinder/templates/deployment-volume.yaml @@ -15,6 +15,8 @@ limitations under the License. {{- if .Values.manifests.deployment_volume }} {{- $envAll := . }} +{{- $internal_ceph_backend := .Values.ceph_client.internal_ceph_backend }} + {{- $mounts_cinder_volume := .Values.pod.mounts.cinder_volume.cinder_volume }} {{- $mounts_cinder_volume_init := .Values.pod.mounts.cinder_volume.init_container }} @@ -58,6 +60,7 @@ spec: {{ tuple $envAll "volume" $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} {{- range $name := rest (splitList "," (include "cinder.utils.ceph_backend_list" $envAll)) }} {{- $backend := index $envAll.Values.conf.backends $name }} + {{- if eq $internal_ceph_backend $name }} - name: ceph-keyring-placement-{{$name}} {{ tuple $envAll "cinder_volume" | include "helm-toolkit.snippets.image" | indent 10 }} {{ dict "envAll" $envAll "application" "cinder_volume" "container" "ceph_keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} @@ -79,6 +82,13 @@ spec: mountPath: /tmp/client-keyring subPath: key readOnly: true + {{- if $envAll.Values.ceph_client.enable_external_ceph_backend }} + - name: external-ceph-keyring + mountPath: /tmp/external-ceph-client-keyring + subPath: key + readOnly: true + {{- end }} + {{- end }} {{- end }} {{- if eq ( split "://" .Values.conf.cinder.coordination.backend_url )._0 "file" }} - name: ceph-coordination-volume-perms @@ -165,6 +175,16 @@ spec: mountPath: /tmp/client-keyring subPath: key readOnly: true + {{- if .Values.ceph_client.enable_external_ceph_backend }} + - name: external-ceph-etc + mountPath: /etc/ceph/external-ceph.conf + subPath: ceph.conf + readOnly: true + - name: external-ceph-keyring + mountPath: /tmp/external-ceph-client-keyring + subPath: key + readOnly: true + {{- end }} {{- end }} {{- if eq ( split "://" .Values.conf.cinder.coordination.backend_url )._0 "file" }} - name: cinder-coordination @@ -239,6 +259,15 @@ spec: - name: ceph-keyring secret: secretName: {{ .Values.secrets.rbd.volume | quote }} + {{- if .Values.ceph_client.enable_external_ceph_backend }} + - name: external-ceph-etc + configMap: + name: {{ .Values.ceph_client.external_ceph.configmap }} + defaultMode: 0444 + - name: external-ceph-keyring + secret: + secretName: {{ .Values.ceph_client.external_ceph.cinder_user_secret_name | quote }} + {{ end }} {{ end }} {{- if eq ( split "://" .Values.conf.cinder.coordination.backend_url )._0 "file" }} # NOTE (portdirect): this will need to be set to a shared mount amongst all cinder diff --git a/cinder/templates/job-storage-init.yaml b/cinder/templates/job-storage-init.yaml index fce9bd4a10..75880209b0 100755 --- a/cinder/templates/job-storage-init.yaml +++ b/cinder/templates/job-storage-init.yaml @@ -15,6 +15,8 @@ limitations under the License. {{- if .Values.manifests.job_storage_init }} {{- $envAll := . }} +{{- $internal_ceph_backend := .Values.ceph_client.internal_ceph_backend }} + {{- $serviceAccountName := "cinder-storage-init" }} {{ tuple $envAll "storage_init" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- @@ -89,6 +91,7 @@ spec: containers: {{- range $name, $backend := .Values.conf.backends }} {{- if (eq "true" ( dict "backend" $backend | include "cinder.utils.is_ceph_backend" )) }} + {{- if eq $internal_ceph_backend $name }} - name: cinder-storage-init-{{$name}} {{ tuple $envAll "cinder_storage_init" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.jobs.storage_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} @@ -134,6 +137,7 @@ spec: subPath: key readOnly: true {{- end }} + {{- end }} {{- end }} {{- end }} volumes: diff --git a/cinder/values.yaml b/cinder/values.yaml index fb09fcd89d..f145a101cc 100644 --- a/cinder/values.yaml +++ b/cinder/values.yaml @@ -376,8 +376,17 @@ network: port: 30877 ceph_client: + # enable this when there is a need to create second ceph backed pointing + # to external ceph cluster + enable_external_ceph_backend: false + # change this in case of first ceph backend name pointing to internal ceph cluster + # is diffrent + internal_ceph_backend: rbd1 configmap: ceph-etc user_secret_name: pvc-ceph-client-key + external_ceph: + configmap: ceph-etc + cinder_user_secret_name: cinder-volume-rbd-keyring conf: paste: