Add cinder RBD support for multiple ceph clusters

The CinderRbdMultiConfig parameter provides a mechanism for
configuring cinder RBD backends associated with external ceph
clusters defined by CephExternalMultiConfig.

A new nova_libvirt_init_secret.sh script handles the creation of
the libvirt secret that is required for nova to connect to volumes
on the cinder RBD backends.

Depends-On: I040e25341c9869ad289d7e7c98e831caef23fece
Change-Id: I73af5b868de629870a35d38f8436e7025aae791e

ci/environments/scenario001-standalone.yaml

@@ -109,6 +109,16 @@ parameter_defaults:
       dashboard_enabled: false
   NovaEnableRbdBackend: true
   CinderEnableRbdBackend: true
+  CinderRbdBackendName: tripleo_ceph,tripleo_ceph2,tripleo_ceph3
+  CinderRbdMultiConfig:
+    tripleo_ceph2:
+      CephClusterName: ceph2
+      CephClusterFSID: af25554b-42f6-4d2b-9b9b-d08a1132d3e8
+      CinderRbdAvailabilityZone: ceph2-AZ
+    tripleo_ceph3:
+      CephClusterName: ceph3
+      CephClusterFSID: e2cba068-5f14-4b0f-b047-acf375c0004a
+      CinderRbdAvailabilityZone: ceph3-AZ
   CinderBackupBackend: ceph
   GlanceBackend: cinder
   GlanceStoreDescription: 'Cinder glance store'

container_config_scripts/nova_libvirt_init_secret.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+set -e
+
+CEPH_INFO=($*)
+
+if [ -z "$CEPH_INFO" ]; then
+    echo "error: At least one CLUSTER:CLIENT tuple must be specified"
+    exit 1
+fi
+
+echo "------------------------------------------------"
+echo "Initializing virsh secrets for: ${CEPH_INFO[@]}"
+
+for INFO in ${CEPH_INFO[@]}; do
+    IFS=: read CLUSTER CLIENT <<< $INFO
+    FSID=$(awk '$1 == "fsid" {print $3}' /etc/ceph/${CLUSTER}.conf)
+
+    echo "--------"
+    echo "Initializing the virsh secret for '$CLUSTER' cluster ($FSID) '$CLIENT' client"
+
+    # Ensure the secret XML file exists. Puppet should have created a secret.xml
+    # file for the first cluster's secret, so detect when to use that file.
+    if grep -q $FSID /etc/nova/secret.xml; then
+        SECRET_FILE="/etc/nova/secret.xml"
+        SECRET_NAME="client.${CLIENT} secret"
+    else
+        SECRET_FILE="/etc/nova/${CLUSTER}-secret.xml"
+        SECRET_NAME="${CLUSTER}.client.${CLIENT} secret"
+    fi
+
+    if [ ! -f $SECRET_FILE ]; then
+        echo "Creating $SECRET_FILE"
+        cat <<EOF > $SECRET_FILE
+<secret ephemeral='no' private='no'>
+  <usage type='ceph'>
+    <name>${SECRET_NAME}</name>
+  </usage>
+  <uuid>${FSID}</uuid>
+</secret>
+EOF
+    else
+        echo "The $SECRET_FILE file already exists"
+    fi
+
+    # Ensure the libvirt secret is defined
+    if /usr/bin/virsh secret-list | grep -q $FSID; then
+        echo "The virsh secret for $FSID has already been defined"
+    else
+        /usr/bin/virsh secret-define --file $SECRET_FILE
+    fi
+
+    # Fetch the key from the keyring and ensure the secret is set
+    KEY=$(awk '$1 == "key" {print $3}' /etc/ceph/${CLUSTER}.client.${CLIENT}.keyring)
+    if /usr/bin/virsh secret-get-value $FSID 2>/dev/null | grep -q $KEY; then
+        echo "The virsh secret for $FSID has already been set"
+    else
+        /usr/bin/virsh secret-set-value --secret $FSID --base64 $KEY
+    fi
+done

deployment/cinder/cinder-common-container-puppet.yaml

@@ -87,6 +87,23 @@ parameters:
       description: >
         The Ceph cluster name must be at least 1 character and contain only
         letters and numbers.
+  CinderRbdMultiConfig:
+    type: json
+    default: {}
+    description: |
+      Dictionary of settings when configuring multiple RBD backends. The
+      hash key is the backend name, and the value is a dictionary of parameter
+      values unique to that backend. The following parameters are required,
+      and must match the corresponding value defined in CephExternalMultiConfig.
+        CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
+        CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
+      The following parameters are optional, and override the corresponding
+      parameter's default value.
+        CephClientUserName
+        CinderRbdPoolName
+        CinderRbdExtraPools
+        CinderRbdAvailabilityZone
+        CinderRbdFlattenVolumeFromSnapshot
 
 conditions:
   cvol_active_active_tls_enabled:
@@ -158,21 +175,35 @@ outputs:
   cinder_common_kolla_permissions:
     description: Common kolla permissions for cinder-volume and cinder-backup services
     value:
-      - path: /var/log/cinder
-        owner: cinder:cinder
-        recurse: true
-      - path:
-          str_replace:
-            template: /etc/ceph/CLUSTER.client.USER.keyring
-            params:
-              CLUSTER: {get_param: CephClusterName}
-              USER: {get_param: CephClientUserName}
-        owner: cinder:cinder
-        perm: '0600'
-      - path: /etc/pki/tls/certs/etcd.crt
-        owner: cinder:cinder
-      - path: /etc/pki/tls/private/etcd.key
-        owner: cinder:cinder
+      list_concat:
+        -
+          - path: /var/log/cinder
+            owner: cinder:cinder
+            recurse: true
+          - path:
+              str_replace:
+                template: /etc/ceph/CLUSTER.client.USER.keyring
+                params:
+                  CLUSTER: {get_param: CephClusterName}
+                  USER: {get_param: CephClientUserName}
+            owner: cinder:cinder
+            perm: '0600'
+          - path: /etc/pki/tls/certs/etcd.crt
+            owner: cinder:cinder
+          - path: /etc/pki/tls/private/etcd.key
+            owner: cinder:cinder
+        - repeat:
+            template:
+              path: /etc/ceph/<%keyring%>
+              owner: cinder:cinder
+              perm: '0600'
+            for_each:
+              <%keyring%>:
+                yaql:
+                  expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u)))
+                  data:
+                    default_user: {get_param: CephClientUserName}
+                    multiconfig: {get_param: CinderRbdMultiConfig}
 
   cinder_volume_host_prep_tasks:
     description: Host prep tasks for the cinder-volume service (HA or non-HA)

deployment/cinder/cinder-volume-container-puppet.yaml

@@ -127,6 +127,10 @@ parameters:
       Valid values are 'auto', 'true' or 'false'. Effective when
       CinderEnableNfsBackend is true.
     type: string
+  CinderRbdBackendName:
+    type: comma_delimited_list
+    default: 'tripleo_ceph'
+    description: A list of Cinder RBD backend names.
   CinderRbdAvailabilityZone:
     default: ''
     description: >
@@ -144,13 +148,29 @@ parameters:
       list. This is in addition to the standard RBD backend driver
       associated with the CinderRbdPoolName.
     type: comma_delimited_list
-
   CinderRbdFlattenVolumeFromSnapshot:
     default: false
     description: >
       Whether RBD volumes created from a snapshot should be flattened
       in order to remove a dependency on the snapshot.
     type: boolean
+  CinderRbdMultiConfig:
+    type: json
+    default: {}
+    description: |
+      Dictionary of settings when configuring multiple RBD backends. The
+      hash key is the backend name, and the value is a dictionary of parameter
+      values unique to that backend. The following parameters are required,
+      and must match the corresponding value defined in CephExternalMultiConfig.
+        CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
+        CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
+      The following parameters are optional, and override the corresponding
+      parameter's default value.
+        CephClientUserName
+        CinderRbdPoolName
+        CinderRbdExtraPools
+        CinderRbdAvailabilityZone
+        CinderRbdFlattenVolumeFromSnapshot
   CephClusterFSID:
     type: string
     description: The Ceph cluster FSID. Must be a UUID.
@@ -229,6 +249,7 @@ outputs:
             tripleo::profile::base::cinder::volume::nfs::cinder_nas_secure_file_permissions: {get_param: CinderNasSecureFilePermissions}
             tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_helper: {get_param: CinderISCSIHelper}
             tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_protocol: {get_param: CinderISCSIProtocol}
+            tripleo::profile::base::cinder::volume::rbd::backend_name: {get_param: CinderRbdBackendName}
             tripleo::profile::base::cinder::volume::rbd::cinder_rbd_ceph_conf:
               list_join:
               - ''
@@ -239,8 +260,8 @@ outputs:
             tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools: {get_param: CinderRbdExtraPools}
             tripleo::profile::base::cinder::volume::rbd::cinder_rbd_secret_uuid: {get_param: CephClusterFSID}
             tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: {get_param: CephClientUserName}
-            tripleo::profile::base::cinder::volume::cinder_rbd_ceph_conf_path: {get_param: CephConfigPath}
             tripleo::profile::base::cinder::volume::rbd::cinder_rbd_flatten_volume_from_snapshot: {get_param: CinderRbdFlattenVolumeFromSnapshot}
+            tripleo::profile::base::cinder::volume::rbd::multi_config: {get_param: CinderRbdMultiConfig}
             # NOTE: bind IP is found in hiera replacing the network name with the local node IP
             # for the given network; replacement examples (eg. for internal_api):
             # internal_api -> IP

deployment/nova/nova-compute-container-puppet.yaml

@@ -59,6 +59,23 @@ parameters:
     default: "/var/lib/tripleo-config/ceph"
     description: |
       The path where the Ceph Cluster config files are stored on the host.
+  CinderRbdMultiConfig:
+    type: json
+    default: {}
+    description: |
+      Dictionary of settings when configuring multiple RBD backends. The
+      hash key is the backend name, and the value is a dictionary of parameter
+      values unique to that backend. The following parameters are required,
+      and must match the corresponding value defined in CephExternalMultiConfig.
+        CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
+        CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
+      The following parameters are optional, and override the corresponding
+      parameter's default value.
+        CephClientUserName
+        CinderRbdPoolName
+        CinderRbdExtraPools
+        CinderRbdAvailabilityZone
+        CinderRbdFlattenVolumeFromSnapshot
   NovaComputeOptVolumes:
     default: []
     description: list of optional volumes to be mounted
@@ -1269,6 +1286,18 @@ outputs:
                         data:
                           user: {get_param: CephClientUserName}
                           multistore: {get_param: GlanceMultistoreConfig}
+              - repeat:
+                  template:
+                    path: /etc/ceph/<%keyring%>
+                    owner: nova:nova
+                    perm: '0600'
+                  for_each:
+                    <%keyring%>:
+                      yaql:
+                        expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u)))
+                        data:
+                          default_user: {get_param: CephClientUserName}
+                          multiconfig: {get_param: CinderRbdMultiConfig}
       container_config_scripts:
         map_merge:
           - {get_attr: [ContainersCommon, container_config_scripts]}

deployment/nova/nova-libvirt-container-puppet.yaml

@@ -84,6 +84,23 @@ parameters:
       description: >
         The Ceph cluster name must be at least 1 character and contain only
         letters and numbers.
+  CinderRbdMultiConfig:
+    type: json
+    default: {}
+    description: |
+      Dictionary of settings when configuring multiple RBD backends. The
+      hash key is the backend name, and the value is a dictionary of parameter
+      values unique to that backend. The following parameters are required,
+      and must match the corresponding value defined in CephExternalMultiConfig.
+        CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
+        CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
+      The following parameters are optional, and override the corresponding
+      parameter's default value.
+        CephClientUserName
+        CinderRbdPoolName
+        CinderRbdExtraPools
+        CinderRbdAvailabilityZone
+        CinderRbdFlattenVolumeFromSnapshot
   UseTLSTransportForVnc:
     type: boolean
     default: true
@@ -505,6 +522,18 @@ outputs:
                         USER: {get_param: CephClientUserName}
                   owner: nova:nova
                   perm: '0600'
+              - repeat:
+                  template:
+                    path: /etc/ceph/<%keyring%>
+                    owner: nova:nova
+                    perm: '0600'
+                  for_each:
+                    <%keyring%>:
+                      yaql:
+                        expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u)))
+                        data:
+                          default_user: {get_param: CephClientUserName}
+                          multiconfig: {get_param: CinderRbdMultiConfig}
         /var/lib/kolla/config_files/nova_virtlogd.json:
           command: /usr/sbin/virtlogd --config /etc/libvirt/virtlogd.conf
           config_files:
@@ -530,6 +559,9 @@ outputs:
                     - use_tls_for_live_migration
                     - '--listen'
                     - ''
+        nova_libvirt_init_secret.sh:
+          mode: "0755"
+          content: { get_file: ../../container_config_scripts/nova_libvirt_init_secret.sh }
       docker_config:
         step_3:
           nova_virtlogd:
@@ -671,18 +703,36 @@ outputs:
                   list_concat:
                     - {get_attr: [ContainersCommon, volumes]}
                     -
-                      - /var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova:ro
+                      - /var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova
                       - /etc/libvirt:/etc/libvirt
                       - /run/libvirt:/run/libvirt:shared
                       - /var/lib/libvirt:/var/lib/libvirt:shared
+                      - /var/lib/container-config-scripts/nova_libvirt_init_secret.sh:/nova_libvirt_init_secret.sh:ro
+                      -
+                        str_replace:
+                          template: HOST_CEPH:/etc/ceph:ro
+                          params:
+                            HOST_CEPH: {get_param: CephConfigPath}
                 command:
-                  - /bin/bash
-                  - -c
-                  - str_replace:
-                      template: /usr/bin/virsh secret-define --file /etc/nova/secret.xml && /usr/bin/virsh secret-set-value --secret 'SECRET_UUID' --base64 'SECRET_KEY'
-                      params:
-                        SECRET_UUID: {get_param: CephClusterFSID}
-                        SECRET_KEY: {get_param: CephClientKey}
+                  list_join:
+                    - ' '
+                    - - str_replace:
+                          template:
+                            "/nova_libvirt_init_secret.sh CLUSTER:USER"
+                          params:
+                            CLUSTER: {get_param: CephClusterName}
+                            USER: {get_param: CephClientUserName}
+                    - repeat:
+                        template:
+                          <%ceph_info%>
+                        for_each:
+                          <%ceph_info%>:
+                            yaql:
+                              expression:
+                                let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}:{1}".format($.CephClusterName, $.get("CephClientUserName", $u)))
+                              data:
+                                default_user: {get_param: CephClientUserName}
+                                multiconfig: {get_param: CinderRbdMultiConfig}
             - {}
       deploy_steps_tasks:
         list_concat:

releasenotes/notes/cinder-rbd-multiconfig-dff6b46a0b20331a.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    A new ``CinderRbdMultiConfig`` parameter may be used to configure
+    additional cinder RBD backends on external Ceph clusters defined by the
+    ``CephExternalMultiConfig`` parameter.