Cinder support multiple Ceph volume backends

Wrap code making the assumption there is only one Ceph backend
(using is_ceph_volume_configured) in a "range" and use
ceph_backend_list helper to iterate all available Ceph backends.

Move Ceph pool application name in values.yaml from
conf.software.rbd.rbd_pool_app_name* to conf.ceph.pools.*.app_name

Change-Id: If1126e51fe9ebb85185e375dc282e83db63d934c
Depends-On: Iaa67061b05a9d355228ad7d3f5ee0f4f04dbdc66
Signed-off-by: Daniel Badea <daniel.badea@windriver.com>

cinder/templates/deployment-backup.yaml

@@ -78,8 +78,9 @@ spec:
               subPath: key
               readOnly: true
         {{ end }}
-        {{- if include "cinder.utils.has_ceph_backend" $envAll }}
+        {{- range $name := rest (splitList "," (include "cinder.utils.ceph_backend_list" $envAll)) }}
-        - name: ceph-keyring-placement
+          {{- $backend := index $envAll.Values.conf.backends $name }}
+        - name: ceph-keyring-placement-{{$name}}
 {{ tuple $envAll "cinder_backup" | include "helm-toolkit.snippets.image" | indent 10 }}
           securityContext:
             runAsUser: 0
@@ -87,7 +88,7 @@ spec:
             - /tmp/ceph-keyring.sh
           env:
             - name: RBD_USER
-              value: {{ index (index .Values.conf.backends (include "cinder.utils.ceph_volume_section_name" $envAll)) "rbd_user" | quote }}
+              value: {{ $backend.rbd_user | quote }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
@@ -101,7 +102,7 @@ spec:
               mountPath: /tmp/client-keyring
               subPath: key
               readOnly: true
-        {{ end }}
+        {{- end }}
         {{- if (contains "cinder.backup.drivers.posix" .Values.conf.cinder.DEFAULT.backup_driver) }}
         - name: ceph-backup-volume-perms
 {{ tuple $envAll "cinder_backup" | include "helm-toolkit.snippets.image" | indent 10 }}

cinder/templates/deployment-volume.yaml

@@ -54,8 +54,9 @@ spec:
         {{ .Values.labels.volume.node_selector_key }}: {{ .Values.labels.volume.node_selector_value }}
       initContainers:
 {{ tuple $envAll "volume" $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
-        {{- if include "cinder.utils.has_ceph_backend" $envAll }}
+        {{- range $name := rest (splitList "," (include "cinder.utils.ceph_backend_list" $envAll)) }}
-        - name: ceph-keyring-placement
+          {{- $backend := index $envAll.Values.conf.backends $name }}
+        - name: ceph-keyring-placement-{{$name}}
 {{ tuple $envAll "cinder_volume" | include "helm-toolkit.snippets.image" | indent 10 }}
           securityContext:
             runAsUser: 0
@@ -63,7 +64,7 @@ spec:
             - /tmp/ceph-keyring.sh
           env:
             - name: RBD_USER
-              value: {{ index (index .Values.conf.backends (include "cinder.utils.ceph_volume_section_name" $envAll)) "rbd_user" | quote }}
+              value: {{ $backend.rbd_user | quote }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
@@ -77,7 +78,7 @@ spec:
               mountPath: /tmp/client-keyring
               subPath: key
               readOnly: true
-        {{ end }}
+        {{- end }}
         {{- if eq ( split "://" .Values.conf.cinder.coordination.backend_url )._0 "file" }}
         - name: ceph-coordination-volume-perms
 {{ tuple $envAll "cinder_volume" | include "helm-toolkit.snippets.image" | indent 10 }}

cinder/templates/job-backup-storage-init.yaml

@@ -112,7 +112,7 @@ spec:
             - name: RBD_POOL_NAME
               value: {{ .Values.conf.cinder.DEFAULT.backup_ceph_pool | quote }}
             - name: RBD_POOL_APP_NAME
-              value: {{ .Values.conf.software.rbd.rbd_pool_app_name_backup | quote }}
+              value: {{ .Values.conf.ceph.pools.backup.app_name | quote }}
             - name: RBD_POOL_USER
               value: {{ .Values.conf.cinder.DEFAULT.backup_ceph_user | quote }}
             - name: RBD_POOL_CRUSH_RULE

cinder/templates/job-storage-init.yaml

@@ -89,7 +89,9 @@ spec:
             {{ end }}
         {{ end }}
       containers:
-        - name: cinder-storage-init
+        {{- range $name, $backend := .Values.conf.backends }}
+          {{- if kindIs "map" $backend }}
+        - name: cinder-storage-init-{{$name}}
 {{ tuple $envAll "cinder_storage_init" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.storage_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           env:
@@ -97,23 +99,23 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-            {{- if include "cinder.utils.has_ceph_backend" $envAll }}
+            {{- if include "cinder.utils.is_ceph_backend" $backend }}
             - name: STORAGE_BACKEND
-              value: {{ index (index .Values.conf.backends (include "cinder.utils.ceph_volume_section_name" $envAll)) "volume_driver" | quote }}
+              value: {{ $backend.volume_driver | quote }}
             - name: RBD_POOL_NAME
-              value: {{ index (index .Values.conf.backends (include "cinder.utils.ceph_volume_section_name" $envAll)) "rbd_pool" | quote }}
+              value: {{ $backend.rbd_pool | quote }}
             - name: RBD_POOL_APP_NAME
-              value: {{ .Values.conf.software.rbd.rbd_pool_app_name | quote }}
+              value: {{ (index $envAll.Values.conf.ceph.pools $backend.rbd_pool).app_name | quote }}
             - name: RBD_POOL_USER
-              value: {{ index (index .Values.conf.backends (include "cinder.utils.ceph_volume_section_name" $envAll)) "rbd_user" | quote }}
+              value: {{ $backend.rbd_user | quote }}
             - name: RBD_POOL_CRUSH_RULE
-              value: {{ .Values.conf.ceph.pools.volume.crush_rule | quote }}
+              value: {{ (index $envAll.Values.conf.ceph.pools $backend.rbd_pool).crush_rule | quote }}
             - name: RBD_POOL_REPLICATION
-              value: {{ .Values.conf.ceph.pools.volume.replication | quote }}
+              value: {{ (index $envAll.Values.conf.ceph.pools $backend.rbd_pool).replication | quote }}
             - name: RBD_POOL_CHUNK_SIZE
-              value: {{ .Values.conf.ceph.pools.volume.chunk_size | quote }}
+              value: {{ (index $envAll.Values.conf.ceph.pools $backend.rbd_pool).chunk_size | quote }}
             - name: RBD_POOL_SECRET
-              value: {{ .Values.secrets.rbd.volume | quote }}
+              value: {{ $envAll.Values.secrets.rbd.volume | quote }}
             {{- end }}
           command:
             - /tmp/storage-init.sh
@@ -124,20 +126,22 @@ spec:
               mountPath: /tmp/storage-init.sh
               subPath: storage-init.sh
               readOnly: true
-            {{- if include "cinder.utils.has_ceph_backend" $envAll }}
+            {{- if include "cinder.utils.is_ceph_backend" $backend }}
             - name: etcceph
               mountPath: /etc/ceph
             - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
-            {{- if empty .Values.conf.ceph.admin_keyring }}
+            {{- if empty $envAll.Values.conf.ceph.admin_keyring }}
             - name: ceph-keyring
               mountPath: /tmp/client-keyring
               subPath: key
               readOnly: true
             {{- end }}
             {{- end }}
+        {{- end }}
+      {{- end }}
       volumes:
         - name: pod-tmp
           emptyDir: {}

cinder/templates/utils/_ceph_backend_list.tpl

@@ -0,0 +1,36 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- /*
+    Return string with all ceph backends separated by comma. The list
+    is either empty or it starts with a comma. Assuming "a", "b" and
+    "c" are ceph backends then ceph_backend_list returns ",a,b,c".
+    This means the first element in the returned list representation
+    can always be skipped.
+
+    Usage:
+        range $name := rest (splitList include "cinder.utils.ceph_backend_list" $)
+*/ -}}
+{{- define "cinder.utils.ceph_backend_list" -}}
+  {{- range $name, $backend := .Values.conf.backends -}}
+    {{- if kindIs "map" $backend }}
+      {{- if (eq $backend.volume_driver "cinder.volume.drivers.rbd.RBDDriver") -}}
+        {{- "," -}}
+        {{- $name -}}
+      {{- end -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}

cinder/templates/utils/_is_ceph_backend.tpl

@@ -14,12 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- define "cinder.utils.ceph_volume_section_name" -}}
+{{- define "cinder.utils.is_ceph_backend" -}}
-{{- range $section, $values := .Values.conf.backends -}}
+  {{- if kindIs "map" . -}}
-{{- if kindIs "map" $values -}}
+    {{- eq .volume_driver "cinder.volume.drivers.rbd.RBDDriver" -}}
-{{- if eq $values.volume_driver "cinder.volume.drivers.rbd.RBDDriver" -}}
+  {{- end -}}
-{{ $section }}
-{{- end -}}
-{{- end -}}
-{{- end -}}
 {{- end -}}

cinder/values.yaml

@@ -308,10 +308,6 @@ ceph_client:
   user_secret_name: pvc-ceph-client-key
 
 conf:
-  software:
-    rbd:
-      rbd_pool_app_name_backup: cinder-backup
-      rbd_pool_app_name: cinder-volume
   paste:
     composite:osapi_volume:
       use: call:cinder.api:root_app_factory
@@ -774,10 +770,12 @@ conf:
         replication: 3
         crush_rule: replicated_rule
         chunk_size: 8
-      volume:
+        app_name: cinder-backup
+      cinder.volumes:
         replication: 3
         crush_rule: replicated_rule
         chunk_size: 8
+        app_name: cinder-volume
   cinder:
     DEFAULT:
       resource_query_filters_file: /etc/cinder/resource_filters.json

releasenotes/notes/rename-ceph-rbd-pool-app-name.yaml

@@ -0,0 +1,7 @@
+---
+other:
+  - |
+    rbd_pool_app_name is a Ceph pool attribute. Moving it from conf.software.rbd to
+    conf.ceph.pools as app_name. This means that conf.software.rbd.rbd_pool_app_name
+    is now conf.ceph.pools.cinder.volumes.app_name and conf.software.rbd.rbd_pool_app_name_backup
+    is now conf.ceph.pools.backup.app_name.

tools/deployment/component/cinder/cinder.sh

@@ -28,10 +28,34 @@ conf:
         replication: 1
         crush_rule: same_host
         chunk_size: 8
-      volume:
+        app_name: cinder-backup
+      # default pool used by rbd1 backend
+      cinder.volumes:
         replication: 1
         crush_rule: same_host
         chunk_size: 8
+        app_name: cinder-volume
+      # secondary pool used by rbd2 backend
+      cinder.volumes.gold:
+        replication: 1
+        crush_rule: same_host
+        chunk_size: 8
+        app_name: cinder-volume
+  backends:
+    # add an extra storage backend same values as rbd1 (see
+    # cinder/values.yaml) except for volume_backend_name and rbd_pool
+    rbd2:
+      volume_driver: cinder.volume.drivers.rbd.RBDDriver
+      volume_backend_name: rbd2
+      rbd_pool: cinder.volumes.gold
+      rbd_ceph_conf: "/etc/ceph/ceph.conf"
+      rbd_flatten_volume_from_snapshot: false
+      report_discard_supported: true
+      rbd_max_clone_depth: 5
+      rbd_store_chunk_size: 4
+      rados_connect_timeout: -1
+      rbd_user: cinder
+      rbd_secret_uuid: 457eb676-33da-42ec-9a8c-9293d545c337
 EOF
 helm upgrade --install cinder ./cinder \
   --namespace=openstack \

tools/deployment/developer/ceph/130-cinder.sh

@@ -28,10 +28,12 @@ conf:
         replication: 1
         crush_rule: same_host
         chunk_size: 8
-      volume:
+        app_name: cinder-backup
+      cinder.volumes:
         replication: 1
         crush_rule: same_host
         chunk_size: 8
+        app_name: cinder-volume
 EOF
 helm upgrade --install cinder ./cinder \
   --namespace=openstack \