From 11d7e1d49ab0cf2bfcdae5d4c9266b592687b7a2 Mon Sep 17 00:00:00 2001 From: Daniel Badea Date: Thu, 20 Jun 2019 14:02:33 +0000 Subject: [PATCH] Cinder support multiple Ceph volume backends Wrap code making the assumption there is only one Ceph backend (using is_ceph_volume_configured) in a "range" and use ceph_backend_list helper to iterate all available Ceph backends. Move Ceph pool application name in values.yaml from conf.software.rbd.rbd_pool_app_name* to conf.ceph.pools.*.app_name Change-Id: If1126e51fe9ebb85185e375dc282e83db63d934c Depends-On: Iaa67061b05a9d355228ad7d3f5ee0f4f04dbdc66 Signed-off-by: Daniel Badea --- cinder/templates/deployment-backup.yaml | 9 ++--- cinder/templates/deployment-volume.yaml | 9 ++--- cinder/templates/job-backup-storage-init.yaml | 2 +- cinder/templates/job-storage-init.yaml | 28 ++++++++------- cinder/templates/utils/_ceph_backend_list.tpl | 36 +++++++++++++++++++ ..._section_name.tpl => _is_ceph_backend.tpl} | 12 +++---- cinder/values.yaml | 8 ++--- .../notes/rename-ceph-rbd-pool-app-name.yaml | 7 ++++ tools/deployment/component/cinder/cinder.sh | 26 +++++++++++++- tools/deployment/developer/ceph/130-cinder.sh | 4 ++- 10 files changed, 105 insertions(+), 36 deletions(-) create mode 100644 cinder/templates/utils/_ceph_backend_list.tpl rename cinder/templates/utils/{_ceph_volume_section_name.tpl => _is_ceph_backend.tpl} (68%) create mode 100644 releasenotes/notes/rename-ceph-rbd-pool-app-name.yaml diff --git a/cinder/templates/deployment-backup.yaml b/cinder/templates/deployment-backup.yaml index 835c2597fb..f097f07b88 100755 --- a/cinder/templates/deployment-backup.yaml +++ b/cinder/templates/deployment-backup.yaml @@ -78,8 +78,9 @@ spec: subPath: key readOnly: true {{ end }} - {{- if include "cinder.utils.has_ceph_backend" $envAll }} - - name: ceph-keyring-placement + {{- range $name := rest (splitList "," (include "cinder.utils.ceph_backend_list" $envAll)) }} + {{- $backend := index $envAll.Values.conf.backends $name }} + - name: ceph-keyring-placement-{{$name}} {{ tuple $envAll "cinder_backup" | include "helm-toolkit.snippets.image" | indent 10 }} securityContext: runAsUser: 0 @@ -87,7 +88,7 @@ spec: - /tmp/ceph-keyring.sh env: - name: RBD_USER - value: {{ index (index .Values.conf.backends (include "cinder.utils.ceph_volume_section_name" $envAll)) "rbd_user" | quote }} + value: {{ $backend.rbd_user | quote }} volumeMounts: - name: pod-tmp mountPath: /tmp @@ -101,7 +102,7 @@ spec: mountPath: /tmp/client-keyring subPath: key readOnly: true - {{ end }} + {{- end }} {{- if (contains "cinder.backup.drivers.posix" .Values.conf.cinder.DEFAULT.backup_driver) }} - name: ceph-backup-volume-perms {{ tuple $envAll "cinder_backup" | include "helm-toolkit.snippets.image" | indent 10 }} diff --git a/cinder/templates/deployment-volume.yaml b/cinder/templates/deployment-volume.yaml index 45298d59a3..f0c251e587 100755 --- a/cinder/templates/deployment-volume.yaml +++ b/cinder/templates/deployment-volume.yaml @@ -54,8 +54,9 @@ spec: {{ .Values.labels.volume.node_selector_key }}: {{ .Values.labels.volume.node_selector_value }} initContainers: {{ tuple $envAll "volume" $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - {{- if include "cinder.utils.has_ceph_backend" $envAll }} - - name: ceph-keyring-placement + {{- range $name := rest (splitList "," (include "cinder.utils.ceph_backend_list" $envAll)) }} + {{- $backend := index $envAll.Values.conf.backends $name }} + - name: ceph-keyring-placement-{{$name}} {{ tuple $envAll "cinder_volume" | include "helm-toolkit.snippets.image" | indent 10 }} securityContext: runAsUser: 0 @@ -63,7 +64,7 @@ spec: - /tmp/ceph-keyring.sh env: - name: RBD_USER - value: {{ index (index .Values.conf.backends (include "cinder.utils.ceph_volume_section_name" $envAll)) "rbd_user" | quote }} + value: {{ $backend.rbd_user | quote }} volumeMounts: - name: pod-tmp mountPath: /tmp @@ -77,7 +78,7 @@ spec: mountPath: /tmp/client-keyring subPath: key readOnly: true - {{ end }} + {{- end }} {{- if eq ( split "://" .Values.conf.cinder.coordination.backend_url )._0 "file" }} - name: ceph-coordination-volume-perms {{ tuple $envAll "cinder_volume" | include "helm-toolkit.snippets.image" | indent 10 }} diff --git a/cinder/templates/job-backup-storage-init.yaml b/cinder/templates/job-backup-storage-init.yaml index 25fe6d3635..2d2bde76c4 100644 --- a/cinder/templates/job-backup-storage-init.yaml +++ b/cinder/templates/job-backup-storage-init.yaml @@ -112,7 +112,7 @@ spec: - name: RBD_POOL_NAME value: {{ .Values.conf.cinder.DEFAULT.backup_ceph_pool | quote }} - name: RBD_POOL_APP_NAME - value: {{ .Values.conf.software.rbd.rbd_pool_app_name_backup | quote }} + value: {{ .Values.conf.ceph.pools.backup.app_name | quote }} - name: RBD_POOL_USER value: {{ .Values.conf.cinder.DEFAULT.backup_ceph_user | quote }} - name: RBD_POOL_CRUSH_RULE diff --git a/cinder/templates/job-storage-init.yaml b/cinder/templates/job-storage-init.yaml index 4c13be6ff2..fad5dbdb9d 100755 --- a/cinder/templates/job-storage-init.yaml +++ b/cinder/templates/job-storage-init.yaml @@ -89,7 +89,9 @@ spec: {{ end }} {{ end }} containers: - - name: cinder-storage-init + {{- range $name, $backend := .Values.conf.backends }} + {{- if kindIs "map" $backend }} + - name: cinder-storage-init-{{$name}} {{ tuple $envAll "cinder_storage_init" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.jobs.storage_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} env: @@ -97,23 +99,23 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - {{- if include "cinder.utils.has_ceph_backend" $envAll }} + {{- if include "cinder.utils.is_ceph_backend" $backend }} - name: STORAGE_BACKEND - value: {{ index (index .Values.conf.backends (include "cinder.utils.ceph_volume_section_name" $envAll)) "volume_driver" | quote }} + value: {{ $backend.volume_driver | quote }} - name: RBD_POOL_NAME - value: {{ index (index .Values.conf.backends (include "cinder.utils.ceph_volume_section_name" $envAll)) "rbd_pool" | quote }} + value: {{ $backend.rbd_pool | quote }} - name: RBD_POOL_APP_NAME - value: {{ .Values.conf.software.rbd.rbd_pool_app_name | quote }} + value: {{ (index $envAll.Values.conf.ceph.pools $backend.rbd_pool).app_name | quote }} - name: RBD_POOL_USER - value: {{ index (index .Values.conf.backends (include "cinder.utils.ceph_volume_section_name" $envAll)) "rbd_user" | quote }} + value: {{ $backend.rbd_user | quote }} - name: RBD_POOL_CRUSH_RULE - value: {{ .Values.conf.ceph.pools.volume.crush_rule | quote }} + value: {{ (index $envAll.Values.conf.ceph.pools $backend.rbd_pool).crush_rule | quote }} - name: RBD_POOL_REPLICATION - value: {{ .Values.conf.ceph.pools.volume.replication | quote }} + value: {{ (index $envAll.Values.conf.ceph.pools $backend.rbd_pool).replication | quote }} - name: RBD_POOL_CHUNK_SIZE - value: {{ .Values.conf.ceph.pools.volume.chunk_size | quote }} + value: {{ (index $envAll.Values.conf.ceph.pools $backend.rbd_pool).chunk_size | quote }} - name: RBD_POOL_SECRET - value: {{ .Values.secrets.rbd.volume | quote }} + value: {{ $envAll.Values.secrets.rbd.volume | quote }} {{- end }} command: - /tmp/storage-init.sh @@ -124,20 +126,22 @@ spec: mountPath: /tmp/storage-init.sh subPath: storage-init.sh readOnly: true - {{- if include "cinder.utils.has_ceph_backend" $envAll }} + {{- if include "cinder.utils.is_ceph_backend" $backend }} - name: etcceph mountPath: /etc/ceph - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true - {{- if empty .Values.conf.ceph.admin_keyring }} + {{- if empty $envAll.Values.conf.ceph.admin_keyring }} - name: ceph-keyring mountPath: /tmp/client-keyring subPath: key readOnly: true {{- end }} {{- end }} + {{- end }} + {{- end }} volumes: - name: pod-tmp emptyDir: {} diff --git a/cinder/templates/utils/_ceph_backend_list.tpl b/cinder/templates/utils/_ceph_backend_list.tpl new file mode 100644 index 0000000000..adcd45d99c --- /dev/null +++ b/cinder/templates/utils/_ceph_backend_list.tpl @@ -0,0 +1,36 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- /* + Return string with all ceph backends separated by comma. The list + is either empty or it starts with a comma. Assuming "a", "b" and + "c" are ceph backends then ceph_backend_list returns ",a,b,c". + This means the first element in the returned list representation + can always be skipped. + + Usage: + range $name := rest (splitList include "cinder.utils.ceph_backend_list" $) +*/ -}} +{{- define "cinder.utils.ceph_backend_list" -}} + {{- range $name, $backend := .Values.conf.backends -}} + {{- if kindIs "map" $backend }} + {{- if (eq $backend.volume_driver "cinder.volume.drivers.rbd.RBDDriver") -}} + {{- "," -}} + {{- $name -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/cinder/templates/utils/_ceph_volume_section_name.tpl b/cinder/templates/utils/_is_ceph_backend.tpl similarity index 68% rename from cinder/templates/utils/_ceph_volume_section_name.tpl rename to cinder/templates/utils/_is_ceph_backend.tpl index af16d6a9c1..3d5c3becc1 100644 --- a/cinder/templates/utils/_ceph_volume_section_name.tpl +++ b/cinder/templates/utils/_is_ceph_backend.tpl @@ -14,12 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- define "cinder.utils.ceph_volume_section_name" -}} -{{- range $section, $values := .Values.conf.backends -}} -{{- if kindIs "map" $values -}} -{{- if eq $values.volume_driver "cinder.volume.drivers.rbd.RBDDriver" -}} -{{ $section }} -{{- end -}} -{{- end -}} -{{- end -}} +{{- define "cinder.utils.is_ceph_backend" -}} + {{- if kindIs "map" . -}} + {{- eq .volume_driver "cinder.volume.drivers.rbd.RBDDriver" -}} + {{- end -}} {{- end -}} diff --git a/cinder/values.yaml b/cinder/values.yaml index f527145cb3..6262c1cf05 100644 --- a/cinder/values.yaml +++ b/cinder/values.yaml @@ -308,10 +308,6 @@ ceph_client: user_secret_name: pvc-ceph-client-key conf: - software: - rbd: - rbd_pool_app_name_backup: cinder-backup - rbd_pool_app_name: cinder-volume paste: composite:osapi_volume: use: call:cinder.api:root_app_factory @@ -774,10 +770,12 @@ conf: replication: 3 crush_rule: replicated_rule chunk_size: 8 - volume: + app_name: cinder-backup + cinder.volumes: replication: 3 crush_rule: replicated_rule chunk_size: 8 + app_name: cinder-volume cinder: DEFAULT: resource_query_filters_file: /etc/cinder/resource_filters.json diff --git a/releasenotes/notes/rename-ceph-rbd-pool-app-name.yaml b/releasenotes/notes/rename-ceph-rbd-pool-app-name.yaml new file mode 100644 index 0000000000..b36fe19a7b --- /dev/null +++ b/releasenotes/notes/rename-ceph-rbd-pool-app-name.yaml @@ -0,0 +1,7 @@ +--- +other: + - | + rbd_pool_app_name is a Ceph pool attribute. Moving it from conf.software.rbd to + conf.ceph.pools as app_name. This means that conf.software.rbd.rbd_pool_app_name + is now conf.ceph.pools.cinder.volumes.app_name and conf.software.rbd.rbd_pool_app_name_backup + is now conf.ceph.pools.backup.app_name. diff --git a/tools/deployment/component/cinder/cinder.sh b/tools/deployment/component/cinder/cinder.sh index 6888e66a3e..5d29861068 100755 --- a/tools/deployment/component/cinder/cinder.sh +++ b/tools/deployment/component/cinder/cinder.sh @@ -28,10 +28,34 @@ conf: replication: 1 crush_rule: same_host chunk_size: 8 - volume: + app_name: cinder-backup + # default pool used by rbd1 backend + cinder.volumes: replication: 1 crush_rule: same_host chunk_size: 8 + app_name: cinder-volume + # secondary pool used by rbd2 backend + cinder.volumes.gold: + replication: 1 + crush_rule: same_host + chunk_size: 8 + app_name: cinder-volume + backends: + # add an extra storage backend same values as rbd1 (see + # cinder/values.yaml) except for volume_backend_name and rbd_pool + rbd2: + volume_driver: cinder.volume.drivers.rbd.RBDDriver + volume_backend_name: rbd2 + rbd_pool: cinder.volumes.gold + rbd_ceph_conf: "/etc/ceph/ceph.conf" + rbd_flatten_volume_from_snapshot: false + report_discard_supported: true + rbd_max_clone_depth: 5 + rbd_store_chunk_size: 4 + rados_connect_timeout: -1 + rbd_user: cinder + rbd_secret_uuid: 457eb676-33da-42ec-9a8c-9293d545c337 EOF helm upgrade --install cinder ./cinder \ --namespace=openstack \ diff --git a/tools/deployment/developer/ceph/130-cinder.sh b/tools/deployment/developer/ceph/130-cinder.sh index 1c60dbb0b3..239050ede6 100755 --- a/tools/deployment/developer/ceph/130-cinder.sh +++ b/tools/deployment/developer/ceph/130-cinder.sh @@ -28,10 +28,12 @@ conf: replication: 1 crush_rule: same_host chunk_size: 8 - volume: + app_name: cinder-backup + cinder.volumes: replication: 1 crush_rule: same_host chunk_size: 8 + app_name: cinder-volume EOF helm upgrade --install cinder ./cinder \ --namespace=openstack \