From e2936d7604b665d35bd8e01f237c0bec0ba205e9 Mon Sep 17 00:00:00 2001 From: Alan Bishop Date: Wed, 14 Apr 2021 12:44:45 -0700 Subject: [PATCH] Add cinder RBD support for multiple ceph clusters The CinderRbdMultiConfig parameter provides a mechanism for configuring cinder RBD backends associated with external ceph clusters defined by CephExternalMultiConfig. A new nova_libvirt_init_secret.sh script handles the creation of the libvirt secret that is required for nova to connect to volumes on the cinder RBD backends. Depends-On: I040e25341c9869ad289d7e7c98e831caef23fece Change-Id: I73af5b868de629870a35d38f8436e7025aae791e --- ci/environments/scenario001-standalone.yaml | 10 +++ .../nova_libvirt_init_secret.sh | 60 +++++++++++++++++ .../cinder-common-container-puppet.yaml | 61 ++++++++++++----- .../cinder-volume-container-puppet.yaml | 25 ++++++- .../nova/nova-compute-container-puppet.yaml | 29 ++++++++ .../nova/nova-libvirt-container-puppet.yaml | 66 ++++++++++++++++--- ...nder-rbd-multiconfig-dff6b46a0b20331a.yaml | 6 ++ 7 files changed, 232 insertions(+), 25 deletions(-) create mode 100755 container_config_scripts/nova_libvirt_init_secret.sh create mode 100644 releasenotes/notes/cinder-rbd-multiconfig-dff6b46a0b20331a.yaml diff --git a/ci/environments/scenario001-standalone.yaml b/ci/environments/scenario001-standalone.yaml index dd6a0fff13..42b4ff983e 100644 --- a/ci/environments/scenario001-standalone.yaml +++ b/ci/environments/scenario001-standalone.yaml @@ -109,6 +109,16 @@ parameter_defaults: dashboard_enabled: false NovaEnableRbdBackend: true CinderEnableRbdBackend: true + CinderRbdBackendName: tripleo_ceph,tripleo_ceph2,tripleo_ceph3 + CinderRbdMultiConfig: + tripleo_ceph2: + CephClusterName: ceph2 + CephClusterFSID: af25554b-42f6-4d2b-9b9b-d08a1132d3e8 + CinderRbdAvailabilityZone: ceph2-AZ + tripleo_ceph3: + CephClusterName: ceph3 + CephClusterFSID: e2cba068-5f14-4b0f-b047-acf375c0004a + CinderRbdAvailabilityZone: ceph3-AZ CinderBackupBackend: ceph GlanceBackend: cinder GlanceStoreDescription: 'Cinder glance store' diff --git a/container_config_scripts/nova_libvirt_init_secret.sh b/container_config_scripts/nova_libvirt_init_secret.sh new file mode 100755 index 0000000000..8c43342a53 --- /dev/null +++ b/container_config_scripts/nova_libvirt_init_secret.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +set -e + +CEPH_INFO=($*) + +if [ -z "$CEPH_INFO" ]; then + echo "error: At least one CLUSTER:CLIENT tuple must be specified" + exit 1 +fi + +echo "------------------------------------------------" +echo "Initializing virsh secrets for: ${CEPH_INFO[@]}" + +for INFO in ${CEPH_INFO[@]}; do + IFS=: read CLUSTER CLIENT <<< $INFO + FSID=$(awk '$1 == "fsid" {print $3}' /etc/ceph/${CLUSTER}.conf) + + echo "--------" + echo "Initializing the virsh secret for '$CLUSTER' cluster ($FSID) '$CLIENT' client" + + # Ensure the secret XML file exists. Puppet should have created a secret.xml + # file for the first cluster's secret, so detect when to use that file. + if grep -q $FSID /etc/nova/secret.xml; then + SECRET_FILE="/etc/nova/secret.xml" + SECRET_NAME="client.${CLIENT} secret" + else + SECRET_FILE="/etc/nova/${CLUSTER}-secret.xml" + SECRET_NAME="${CLUSTER}.client.${CLIENT} secret" + fi + + if [ ! -f $SECRET_FILE ]; then + echo "Creating $SECRET_FILE" + cat < $SECRET_FILE + + + ${SECRET_NAME} + + ${FSID} + +EOF + else + echo "The $SECRET_FILE file already exists" + fi + + # Ensure the libvirt secret is defined + if /usr/bin/virsh secret-list | grep -q $FSID; then + echo "The virsh secret for $FSID has already been defined" + else + /usr/bin/virsh secret-define --file $SECRET_FILE + fi + + # Fetch the key from the keyring and ensure the secret is set + KEY=$(awk '$1 == "key" {print $3}' /etc/ceph/${CLUSTER}.client.${CLIENT}.keyring) + if /usr/bin/virsh secret-get-value $FSID 2>/dev/null | grep -q $KEY; then + echo "The virsh secret for $FSID has already been set" + else + /usr/bin/virsh secret-set-value --secret $FSID --base64 $KEY + fi +done diff --git a/deployment/cinder/cinder-common-container-puppet.yaml b/deployment/cinder/cinder-common-container-puppet.yaml index 18c164bf0f..739546936d 100644 --- a/deployment/cinder/cinder-common-container-puppet.yaml +++ b/deployment/cinder/cinder-common-container-puppet.yaml @@ -87,6 +87,23 @@ parameters: description: > The Ceph cluster name must be at least 1 character and contain only letters and numbers. + CinderRbdMultiConfig: + type: json + default: {} + description: | + Dictionary of settings when configuring multiple RBD backends. The + hash key is the backend name, and the value is a dictionary of parameter + values unique to that backend. The following parameters are required, + and must match the corresponding value defined in CephExternalMultiConfig. + CephClusterName (must match the CephExternalMultiConfig entry's 'cluster') + CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid') + The following parameters are optional, and override the corresponding + parameter's default value. + CephClientUserName + CinderRbdPoolName + CinderRbdExtraPools + CinderRbdAvailabilityZone + CinderRbdFlattenVolumeFromSnapshot conditions: cvol_active_active_tls_enabled: @@ -158,21 +175,35 @@ outputs: cinder_common_kolla_permissions: description: Common kolla permissions for cinder-volume and cinder-backup services value: - - path: /var/log/cinder - owner: cinder:cinder - recurse: true - - path: - str_replace: - template: /etc/ceph/CLUSTER.client.USER.keyring - params: - CLUSTER: {get_param: CephClusterName} - USER: {get_param: CephClientUserName} - owner: cinder:cinder - perm: '0600' - - path: /etc/pki/tls/certs/etcd.crt - owner: cinder:cinder - - path: /etc/pki/tls/private/etcd.key - owner: cinder:cinder + list_concat: + - + - path: /var/log/cinder + owner: cinder:cinder + recurse: true + - path: + str_replace: + template: /etc/ceph/CLUSTER.client.USER.keyring + params: + CLUSTER: {get_param: CephClusterName} + USER: {get_param: CephClientUserName} + owner: cinder:cinder + perm: '0600' + - path: /etc/pki/tls/certs/etcd.crt + owner: cinder:cinder + - path: /etc/pki/tls/private/etcd.key + owner: cinder:cinder + - repeat: + template: + path: /etc/ceph/<%keyring%> + owner: cinder:cinder + perm: '0600' + for_each: + <%keyring%>: + yaql: + expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u))) + data: + default_user: {get_param: CephClientUserName} + multiconfig: {get_param: CinderRbdMultiConfig} cinder_volume_host_prep_tasks: description: Host prep tasks for the cinder-volume service (HA or non-HA) diff --git a/deployment/cinder/cinder-volume-container-puppet.yaml b/deployment/cinder/cinder-volume-container-puppet.yaml index f6f5d64a0a..8cf667d14f 100644 --- a/deployment/cinder/cinder-volume-container-puppet.yaml +++ b/deployment/cinder/cinder-volume-container-puppet.yaml @@ -127,6 +127,10 @@ parameters: Valid values are 'auto', 'true' or 'false'. Effective when CinderEnableNfsBackend is true. type: string + CinderRbdBackendName: + type: comma_delimited_list + default: 'tripleo_ceph' + description: A list of Cinder RBD backend names. CinderRbdAvailabilityZone: default: '' description: > @@ -144,13 +148,29 @@ parameters: list. This is in addition to the standard RBD backend driver associated with the CinderRbdPoolName. type: comma_delimited_list - CinderRbdFlattenVolumeFromSnapshot: default: false description: > Whether RBD volumes created from a snapshot should be flattened in order to remove a dependency on the snapshot. type: boolean + CinderRbdMultiConfig: + type: json + default: {} + description: | + Dictionary of settings when configuring multiple RBD backends. The + hash key is the backend name, and the value is a dictionary of parameter + values unique to that backend. The following parameters are required, + and must match the corresponding value defined in CephExternalMultiConfig. + CephClusterName (must match the CephExternalMultiConfig entry's 'cluster') + CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid') + The following parameters are optional, and override the corresponding + parameter's default value. + CephClientUserName + CinderRbdPoolName + CinderRbdExtraPools + CinderRbdAvailabilityZone + CinderRbdFlattenVolumeFromSnapshot CephClusterFSID: type: string description: The Ceph cluster FSID. Must be a UUID. @@ -229,6 +249,7 @@ outputs: tripleo::profile::base::cinder::volume::nfs::cinder_nas_secure_file_permissions: {get_param: CinderNasSecureFilePermissions} tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_helper: {get_param: CinderISCSIHelper} tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_protocol: {get_param: CinderISCSIProtocol} + tripleo::profile::base::cinder::volume::rbd::backend_name: {get_param: CinderRbdBackendName} tripleo::profile::base::cinder::volume::rbd::cinder_rbd_ceph_conf: list_join: - '' @@ -239,8 +260,8 @@ outputs: tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools: {get_param: CinderRbdExtraPools} tripleo::profile::base::cinder::volume::rbd::cinder_rbd_secret_uuid: {get_param: CephClusterFSID} tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: {get_param: CephClientUserName} - tripleo::profile::base::cinder::volume::cinder_rbd_ceph_conf_path: {get_param: CephConfigPath} tripleo::profile::base::cinder::volume::rbd::cinder_rbd_flatten_volume_from_snapshot: {get_param: CinderRbdFlattenVolumeFromSnapshot} + tripleo::profile::base::cinder::volume::rbd::multi_config: {get_param: CinderRbdMultiConfig} # NOTE: bind IP is found in hiera replacing the network name with the local node IP # for the given network; replacement examples (eg. for internal_api): # internal_api -> IP diff --git a/deployment/nova/nova-compute-container-puppet.yaml b/deployment/nova/nova-compute-container-puppet.yaml index 7e17cb0688..7b0a867901 100644 --- a/deployment/nova/nova-compute-container-puppet.yaml +++ b/deployment/nova/nova-compute-container-puppet.yaml @@ -59,6 +59,23 @@ parameters: default: "/var/lib/tripleo-config/ceph" description: | The path where the Ceph Cluster config files are stored on the host. + CinderRbdMultiConfig: + type: json + default: {} + description: | + Dictionary of settings when configuring multiple RBD backends. The + hash key is the backend name, and the value is a dictionary of parameter + values unique to that backend. The following parameters are required, + and must match the corresponding value defined in CephExternalMultiConfig. + CephClusterName (must match the CephExternalMultiConfig entry's 'cluster') + CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid') + The following parameters are optional, and override the corresponding + parameter's default value. + CephClientUserName + CinderRbdPoolName + CinderRbdExtraPools + CinderRbdAvailabilityZone + CinderRbdFlattenVolumeFromSnapshot NovaComputeOptVolumes: default: [] description: list of optional volumes to be mounted @@ -1269,6 +1286,18 @@ outputs: data: user: {get_param: CephClientUserName} multistore: {get_param: GlanceMultistoreConfig} + - repeat: + template: + path: /etc/ceph/<%keyring%> + owner: nova:nova + perm: '0600' + for_each: + <%keyring%>: + yaql: + expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u))) + data: + default_user: {get_param: CephClientUserName} + multiconfig: {get_param: CinderRbdMultiConfig} container_config_scripts: map_merge: - {get_attr: [ContainersCommon, container_config_scripts]} diff --git a/deployment/nova/nova-libvirt-container-puppet.yaml b/deployment/nova/nova-libvirt-container-puppet.yaml index fa48950288..d489f34cd9 100644 --- a/deployment/nova/nova-libvirt-container-puppet.yaml +++ b/deployment/nova/nova-libvirt-container-puppet.yaml @@ -84,6 +84,23 @@ parameters: description: > The Ceph cluster name must be at least 1 character and contain only letters and numbers. + CinderRbdMultiConfig: + type: json + default: {} + description: | + Dictionary of settings when configuring multiple RBD backends. The + hash key is the backend name, and the value is a dictionary of parameter + values unique to that backend. The following parameters are required, + and must match the corresponding value defined in CephExternalMultiConfig. + CephClusterName (must match the CephExternalMultiConfig entry's 'cluster') + CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid') + The following parameters are optional, and override the corresponding + parameter's default value. + CephClientUserName + CinderRbdPoolName + CinderRbdExtraPools + CinderRbdAvailabilityZone + CinderRbdFlattenVolumeFromSnapshot UseTLSTransportForVnc: type: boolean default: true @@ -505,6 +522,18 @@ outputs: USER: {get_param: CephClientUserName} owner: nova:nova perm: '0600' + - repeat: + template: + path: /etc/ceph/<%keyring%> + owner: nova:nova + perm: '0600' + for_each: + <%keyring%>: + yaql: + expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u))) + data: + default_user: {get_param: CephClientUserName} + multiconfig: {get_param: CinderRbdMultiConfig} /var/lib/kolla/config_files/nova_virtlogd.json: command: /usr/sbin/virtlogd --config /etc/libvirt/virtlogd.conf config_files: @@ -530,6 +559,9 @@ outputs: - use_tls_for_live_migration - '--listen' - '' + nova_libvirt_init_secret.sh: + mode: "0755" + content: { get_file: ../../container_config_scripts/nova_libvirt_init_secret.sh } docker_config: step_3: nova_virtlogd: @@ -671,18 +703,36 @@ outputs: list_concat: - {get_attr: [ContainersCommon, volumes]} - - - /var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova:ro + - /var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova - /etc/libvirt:/etc/libvirt - /run/libvirt:/run/libvirt:shared - /var/lib/libvirt:/var/lib/libvirt:shared + - /var/lib/container-config-scripts/nova_libvirt_init_secret.sh:/nova_libvirt_init_secret.sh:ro + - + str_replace: + template: HOST_CEPH:/etc/ceph:ro + params: + HOST_CEPH: {get_param: CephConfigPath} command: - - /bin/bash - - -c - - str_replace: - template: /usr/bin/virsh secret-define --file /etc/nova/secret.xml && /usr/bin/virsh secret-set-value --secret 'SECRET_UUID' --base64 'SECRET_KEY' - params: - SECRET_UUID: {get_param: CephClusterFSID} - SECRET_KEY: {get_param: CephClientKey} + list_join: + - ' ' + - - str_replace: + template: + "/nova_libvirt_init_secret.sh CLUSTER:USER" + params: + CLUSTER: {get_param: CephClusterName} + USER: {get_param: CephClientUserName} + - repeat: + template: + <%ceph_info%> + for_each: + <%ceph_info%>: + yaql: + expression: + let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}:{1}".format($.CephClusterName, $.get("CephClientUserName", $u))) + data: + default_user: {get_param: CephClientUserName} + multiconfig: {get_param: CinderRbdMultiConfig} - {} deploy_steps_tasks: list_concat: diff --git a/releasenotes/notes/cinder-rbd-multiconfig-dff6b46a0b20331a.yaml b/releasenotes/notes/cinder-rbd-multiconfig-dff6b46a0b20331a.yaml new file mode 100644 index 0000000000..ba8e42d958 --- /dev/null +++ b/releasenotes/notes/cinder-rbd-multiconfig-dff6b46a0b20331a.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + A new ``CinderRbdMultiConfig`` parameter may be used to configure + additional cinder RBD backends on external Ceph clusters defined by the + ``CephExternalMultiConfig`` parameter.