Add cinder RBD support for multiple ceph clusters

The CinderRbdMultiConfig parameter provides a mechanism for
configuring cinder RBD backends associated with external ceph
clusters defined by CephExternalMultiConfig.

A new nova_libvirt_init_secret.sh script handles the creation of
the libvirt secret that is required for nova to connect to volumes
on the cinder RBD backends.

Depends-On: I040e25341c9869ad289d7e7c98e831caef23fece
Change-Id: I73af5b868de629870a35d38f8436e7025aae791e
This commit is contained in:
Alan Bishop 2021-04-14 12:44:45 -07:00
parent c6c513a96a
commit e2936d7604
7 changed files with 232 additions and 25 deletions

View File

@ -109,6 +109,16 @@ parameter_defaults:
dashboard_enabled: false
NovaEnableRbdBackend: true
CinderEnableRbdBackend: true
CinderRbdBackendName: tripleo_ceph,tripleo_ceph2,tripleo_ceph3
CinderRbdMultiConfig:
tripleo_ceph2:
CephClusterName: ceph2
CephClusterFSID: af25554b-42f6-4d2b-9b9b-d08a1132d3e8
CinderRbdAvailabilityZone: ceph2-AZ
tripleo_ceph3:
CephClusterName: ceph3
CephClusterFSID: e2cba068-5f14-4b0f-b047-acf375c0004a
CinderRbdAvailabilityZone: ceph3-AZ
CinderBackupBackend: ceph
GlanceBackend: cinder
GlanceStoreDescription: 'Cinder glance store'

View File

@ -0,0 +1,60 @@
#!/bin/bash
set -e
CEPH_INFO=($*)
if [ -z "$CEPH_INFO" ]; then
echo "error: At least one CLUSTER:CLIENT tuple must be specified"
exit 1
fi
echo "------------------------------------------------"
echo "Initializing virsh secrets for: ${CEPH_INFO[@]}"
for INFO in ${CEPH_INFO[@]}; do
IFS=: read CLUSTER CLIENT <<< $INFO
FSID=$(awk '$1 == "fsid" {print $3}' /etc/ceph/${CLUSTER}.conf)
echo "--------"
echo "Initializing the virsh secret for '$CLUSTER' cluster ($FSID) '$CLIENT' client"
# Ensure the secret XML file exists. Puppet should have created a secret.xml
# file for the first cluster's secret, so detect when to use that file.
if grep -q $FSID /etc/nova/secret.xml; then
SECRET_FILE="/etc/nova/secret.xml"
SECRET_NAME="client.${CLIENT} secret"
else
SECRET_FILE="/etc/nova/${CLUSTER}-secret.xml"
SECRET_NAME="${CLUSTER}.client.${CLIENT} secret"
fi
if [ ! -f $SECRET_FILE ]; then
echo "Creating $SECRET_FILE"
cat <<EOF > $SECRET_FILE
<secret ephemeral='no' private='no'>
<usage type='ceph'>
<name>${SECRET_NAME}</name>
</usage>
<uuid>${FSID}</uuid>
</secret>
EOF
else
echo "The $SECRET_FILE file already exists"
fi
# Ensure the libvirt secret is defined
if /usr/bin/virsh secret-list | grep -q $FSID; then
echo "The virsh secret for $FSID has already been defined"
else
/usr/bin/virsh secret-define --file $SECRET_FILE
fi
# Fetch the key from the keyring and ensure the secret is set
KEY=$(awk '$1 == "key" {print $3}' /etc/ceph/${CLUSTER}.client.${CLIENT}.keyring)
if /usr/bin/virsh secret-get-value $FSID 2>/dev/null | grep -q $KEY; then
echo "The virsh secret for $FSID has already been set"
else
/usr/bin/virsh secret-set-value --secret $FSID --base64 $KEY
fi
done

View File

@ -87,6 +87,23 @@ parameters:
description: >
The Ceph cluster name must be at least 1 character and contain only
letters and numbers.
CinderRbdMultiConfig:
type: json
default: {}
description: |
Dictionary of settings when configuring multiple RBD backends. The
hash key is the backend name, and the value is a dictionary of parameter
values unique to that backend. The following parameters are required,
and must match the corresponding value defined in CephExternalMultiConfig.
CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
The following parameters are optional, and override the corresponding
parameter's default value.
CephClientUserName
CinderRbdPoolName
CinderRbdExtraPools
CinderRbdAvailabilityZone
CinderRbdFlattenVolumeFromSnapshot
conditions:
cvol_active_active_tls_enabled:
@ -158,21 +175,35 @@ outputs:
cinder_common_kolla_permissions:
description: Common kolla permissions for cinder-volume and cinder-backup services
value:
- path: /var/log/cinder
owner: cinder:cinder
recurse: true
- path:
str_replace:
template: /etc/ceph/CLUSTER.client.USER.keyring
params:
CLUSTER: {get_param: CephClusterName}
USER: {get_param: CephClientUserName}
owner: cinder:cinder
perm: '0600'
- path: /etc/pki/tls/certs/etcd.crt
owner: cinder:cinder
- path: /etc/pki/tls/private/etcd.key
owner: cinder:cinder
list_concat:
-
- path: /var/log/cinder
owner: cinder:cinder
recurse: true
- path:
str_replace:
template: /etc/ceph/CLUSTER.client.USER.keyring
params:
CLUSTER: {get_param: CephClusterName}
USER: {get_param: CephClientUserName}
owner: cinder:cinder
perm: '0600'
- path: /etc/pki/tls/certs/etcd.crt
owner: cinder:cinder
- path: /etc/pki/tls/private/etcd.key
owner: cinder:cinder
- repeat:
template:
path: /etc/ceph/<%keyring%>
owner: cinder:cinder
perm: '0600'
for_each:
<%keyring%>:
yaql:
expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u)))
data:
default_user: {get_param: CephClientUserName}
multiconfig: {get_param: CinderRbdMultiConfig}
cinder_volume_host_prep_tasks:
description: Host prep tasks for the cinder-volume service (HA or non-HA)

View File

@ -127,6 +127,10 @@ parameters:
Valid values are 'auto', 'true' or 'false'. Effective when
CinderEnableNfsBackend is true.
type: string
CinderRbdBackendName:
type: comma_delimited_list
default: 'tripleo_ceph'
description: A list of Cinder RBD backend names.
CinderRbdAvailabilityZone:
default: ''
description: >
@ -144,13 +148,29 @@ parameters:
list. This is in addition to the standard RBD backend driver
associated with the CinderRbdPoolName.
type: comma_delimited_list
CinderRbdFlattenVolumeFromSnapshot:
default: false
description: >
Whether RBD volumes created from a snapshot should be flattened
in order to remove a dependency on the snapshot.
type: boolean
CinderRbdMultiConfig:
type: json
default: {}
description: |
Dictionary of settings when configuring multiple RBD backends. The
hash key is the backend name, and the value is a dictionary of parameter
values unique to that backend. The following parameters are required,
and must match the corresponding value defined in CephExternalMultiConfig.
CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
The following parameters are optional, and override the corresponding
parameter's default value.
CephClientUserName
CinderRbdPoolName
CinderRbdExtraPools
CinderRbdAvailabilityZone
CinderRbdFlattenVolumeFromSnapshot
CephClusterFSID:
type: string
description: The Ceph cluster FSID. Must be a UUID.
@ -229,6 +249,7 @@ outputs:
tripleo::profile::base::cinder::volume::nfs::cinder_nas_secure_file_permissions: {get_param: CinderNasSecureFilePermissions}
tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_helper: {get_param: CinderISCSIHelper}
tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_protocol: {get_param: CinderISCSIProtocol}
tripleo::profile::base::cinder::volume::rbd::backend_name: {get_param: CinderRbdBackendName}
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_ceph_conf:
list_join:
- ''
@ -239,8 +260,8 @@ outputs:
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools: {get_param: CinderRbdExtraPools}
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_secret_uuid: {get_param: CephClusterFSID}
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: {get_param: CephClientUserName}
tripleo::profile::base::cinder::volume::cinder_rbd_ceph_conf_path: {get_param: CephConfigPath}
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_flatten_volume_from_snapshot: {get_param: CinderRbdFlattenVolumeFromSnapshot}
tripleo::profile::base::cinder::volume::rbd::multi_config: {get_param: CinderRbdMultiConfig}
# NOTE: bind IP is found in hiera replacing the network name with the local node IP
# for the given network; replacement examples (eg. for internal_api):
# internal_api -> IP

View File

@ -59,6 +59,23 @@ parameters:
default: "/var/lib/tripleo-config/ceph"
description: |
The path where the Ceph Cluster config files are stored on the host.
CinderRbdMultiConfig:
type: json
default: {}
description: |
Dictionary of settings when configuring multiple RBD backends. The
hash key is the backend name, and the value is a dictionary of parameter
values unique to that backend. The following parameters are required,
and must match the corresponding value defined in CephExternalMultiConfig.
CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
The following parameters are optional, and override the corresponding
parameter's default value.
CephClientUserName
CinderRbdPoolName
CinderRbdExtraPools
CinderRbdAvailabilityZone
CinderRbdFlattenVolumeFromSnapshot
NovaComputeOptVolumes:
default: []
description: list of optional volumes to be mounted
@ -1269,6 +1286,18 @@ outputs:
data:
user: {get_param: CephClientUserName}
multistore: {get_param: GlanceMultistoreConfig}
- repeat:
template:
path: /etc/ceph/<%keyring%>
owner: nova:nova
perm: '0600'
for_each:
<%keyring%>:
yaql:
expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u)))
data:
default_user: {get_param: CephClientUserName}
multiconfig: {get_param: CinderRbdMultiConfig}
container_config_scripts:
map_merge:
- {get_attr: [ContainersCommon, container_config_scripts]}

View File

@ -84,6 +84,23 @@ parameters:
description: >
The Ceph cluster name must be at least 1 character and contain only
letters and numbers.
CinderRbdMultiConfig:
type: json
default: {}
description: |
Dictionary of settings when configuring multiple RBD backends. The
hash key is the backend name, and the value is a dictionary of parameter
values unique to that backend. The following parameters are required,
and must match the corresponding value defined in CephExternalMultiConfig.
CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
The following parameters are optional, and override the corresponding
parameter's default value.
CephClientUserName
CinderRbdPoolName
CinderRbdExtraPools
CinderRbdAvailabilityZone
CinderRbdFlattenVolumeFromSnapshot
UseTLSTransportForVnc:
type: boolean
default: true
@ -505,6 +522,18 @@ outputs:
USER: {get_param: CephClientUserName}
owner: nova:nova
perm: '0600'
- repeat:
template:
path: /etc/ceph/<%keyring%>
owner: nova:nova
perm: '0600'
for_each:
<%keyring%>:
yaql:
expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u)))
data:
default_user: {get_param: CephClientUserName}
multiconfig: {get_param: CinderRbdMultiConfig}
/var/lib/kolla/config_files/nova_virtlogd.json:
command: /usr/sbin/virtlogd --config /etc/libvirt/virtlogd.conf
config_files:
@ -530,6 +559,9 @@ outputs:
- use_tls_for_live_migration
- '--listen'
- ''
nova_libvirt_init_secret.sh:
mode: "0755"
content: { get_file: ../../container_config_scripts/nova_libvirt_init_secret.sh }
docker_config:
step_3:
nova_virtlogd:
@ -671,18 +703,36 @@ outputs:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
-
- /var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova:ro
- /var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova
- /etc/libvirt:/etc/libvirt
- /run/libvirt:/run/libvirt:shared
- /var/lib/libvirt:/var/lib/libvirt:shared
- /var/lib/container-config-scripts/nova_libvirt_init_secret.sh:/nova_libvirt_init_secret.sh:ro
-
str_replace:
template: HOST_CEPH:/etc/ceph:ro
params:
HOST_CEPH: {get_param: CephConfigPath}
command:
- /bin/bash
- -c
- str_replace:
template: /usr/bin/virsh secret-define --file /etc/nova/secret.xml && /usr/bin/virsh secret-set-value --secret 'SECRET_UUID' --base64 'SECRET_KEY'
params:
SECRET_UUID: {get_param: CephClusterFSID}
SECRET_KEY: {get_param: CephClientKey}
list_join:
- ' '
- - str_replace:
template:
"/nova_libvirt_init_secret.sh CLUSTER:USER"
params:
CLUSTER: {get_param: CephClusterName}
USER: {get_param: CephClientUserName}
- repeat:
template:
<%ceph_info%>
for_each:
<%ceph_info%>:
yaql:
expression:
let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}:{1}".format($.CephClusterName, $.get("CephClientUserName", $u)))
data:
default_user: {get_param: CephClientUserName}
multiconfig: {get_param: CinderRbdMultiConfig}
- {}
deploy_steps_tasks:
list_concat:

View File

@ -0,0 +1,6 @@
---
features:
- |
A new ``CinderRbdMultiConfig`` parameter may be used to configure
additional cinder RBD backends on external Ceph clusters defined by the
``CephExternalMultiConfig`` parameter.