[Cinder] Support for external ceph backend

This is to add support for external ceph cluster as second backend
for cinder.

prerequisite:
  - create ceph.conf for external ceph cluster as configmap
  - create cinder2 user and keyring on external ceph cluster
  - create keyring of user cinder2 as a secret

now point configmap and secret created as prerequisite under
values.ceph_client.external_ceph section

also we need to add second backend under values.conf.backends section
with all the details of external ceph cluster.

Change-Id: I8df9f1da7208304f479dbb70b19af97fb01cd38f

cinder/templates/bin/_ceph-keyring.sh.tpl

@@ -22,4 +22,14 @@ cat <<EOF > /etc/ceph/ceph.client.${RBD_USER}.keyring
     key = $(cat /tmp/client-keyring)
 EOF
 
+if [ -f "/tmp/external-ceph-client-keyring" ]; then
+  RBD_USER=cinder2
+  cat <<EOF2 > /etc/ceph/ceph.client.${RBD_USER}.keyring
+[client.${RBD_USER}]
+    key = $(cat /tmp/external-ceph-client-keyring)
+EOF2
+
+fi
+
+
 exit 0

cinder/templates/deployment-volume.yaml

@@ -15,6 +15,8 @@ limitations under the License.
 {{- if .Values.manifests.deployment_volume }}
 {{- $envAll := . }}
 
+{{- $internal_ceph_backend := .Values.ceph_client.internal_ceph_backend }}
+
 {{- $mounts_cinder_volume := .Values.pod.mounts.cinder_volume.cinder_volume }}
 {{- $mounts_cinder_volume_init := .Values.pod.mounts.cinder_volume.init_container }}
 
@@ -58,6 +60,7 @@ spec:
 {{ tuple $envAll "volume" $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
         {{- range $name := rest (splitList "," (include "cinder.utils.ceph_backend_list" $envAll)) }}
           {{- $backend := index $envAll.Values.conf.backends $name }}
+            {{- if eq $internal_ceph_backend $name }}
         - name: ceph-keyring-placement-{{$name}}
 {{ tuple $envAll "cinder_volume" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ dict "envAll" $envAll "application" "cinder_volume" "container" "ceph_keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
@@ -79,6 +82,13 @@ spec:
               mountPath: /tmp/client-keyring
               subPath: key
               readOnly: true
+            {{- if $envAll.Values.ceph_client.enable_external_ceph_backend }}
+            - name: external-ceph-keyring
+              mountPath: /tmp/external-ceph-client-keyring
+              subPath: key
+              readOnly: true
+            {{- end }}
+            {{- end }}
         {{- end }}
         {{- if eq ( split "://" .Values.conf.cinder.coordination.backend_url )._0 "file" }}
         - name: ceph-coordination-volume-perms
@@ -165,6 +175,16 @@ spec:
               mountPath: /tmp/client-keyring
               subPath: key
               readOnly: true
+            {{- if .Values.ceph_client.enable_external_ceph_backend }}
+            - name: external-ceph-etc
+              mountPath: /etc/ceph/external-ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+            - name: external-ceph-keyring
+              mountPath: /tmp/external-ceph-client-keyring
+              subPath: key
+              readOnly: true
+            {{- end }}
             {{- end }}
             {{- if eq ( split "://" .Values.conf.cinder.coordination.backend_url )._0 "file" }}
             - name: cinder-coordination
@@ -239,6 +259,15 @@ spec:
         - name: ceph-keyring
           secret:
             secretName: {{ .Values.secrets.rbd.volume | quote }}
+        {{- if .Values.ceph_client.enable_external_ceph_backend }}
+        - name: external-ceph-etc
+          configMap:
+            name: {{ .Values.ceph_client.external_ceph.configmap }}
+            defaultMode: 0444
+        - name: external-ceph-keyring
+          secret:
+            secretName: {{ .Values.ceph_client.external_ceph.cinder_user_secret_name | quote }}
+        {{ end }}
         {{ end }}
         {{- if eq ( split "://" .Values.conf.cinder.coordination.backend_url )._0 "file" }}
         # NOTE (portdirect): this will need to be set to a shared mount amongst all cinder

cinder/templates/job-storage-init.yaml

@@ -15,6 +15,8 @@ limitations under the License.
 {{- if .Values.manifests.job_storage_init }}
 {{- $envAll := . }}
 
+{{- $internal_ceph_backend := .Values.ceph_client.internal_ceph_backend }}
+
 {{- $serviceAccountName := "cinder-storage-init" }}
 {{ tuple $envAll "storage_init" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
@@ -89,6 +91,7 @@ spec:
       containers:
         {{- range $name, $backend := .Values.conf.backends }}
           {{- if (eq "true" ( dict "backend" $backend | include "cinder.utils.is_ceph_backend" )) }}
+            {{- if eq $internal_ceph_backend $name }}
         - name: cinder-storage-init-{{$name}}
 {{ tuple $envAll "cinder_storage_init" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.storage_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@@ -134,6 +137,7 @@ spec:
               subPath: key
               readOnly: true
             {{- end }}
+            {{- end }}
         {{- end }}
       {{- end }}
       volumes:

cinder/values.yaml

@@ -376,8 +376,17 @@ network:
       port: 30877
 
 ceph_client:
+  # enable this when there is a need to create second ceph backed pointing
+  # to external ceph cluster
+  enable_external_ceph_backend: false
+  # change this in case of first ceph backend name pointing to internal ceph cluster
+  # is diffrent
+  internal_ceph_backend: rbd1
   configmap: ceph-etc
   user_secret_name: pvc-ceph-client-key
+  external_ceph:
+    configmap: ceph-etc
+    cinder_user_secret_name: cinder-volume-rbd-keyring
 
 conf:
   paste: