Add cinder RBD support for multiple ceph clusters
The CinderRbdMultiConfig parameter provides a mechanism for configuring cinder RBD backends associated with external ceph clusters defined by CephExternalMultiConfig. A new nova_libvirt_init_secret.sh script handles the creation of the libvirt secret that is required for nova to connect to volumes on the cinder RBD backends. Depends-On: I040e25341c9869ad289d7e7c98e831caef23fece Change-Id: I73af5b868de629870a35d38f8436e7025aae791e
This commit is contained in:
parent
c6c513a96a
commit
e2936d7604
|
@ -109,6 +109,16 @@ parameter_defaults:
|
|||
dashboard_enabled: false
|
||||
NovaEnableRbdBackend: true
|
||||
CinderEnableRbdBackend: true
|
||||
CinderRbdBackendName: tripleo_ceph,tripleo_ceph2,tripleo_ceph3
|
||||
CinderRbdMultiConfig:
|
||||
tripleo_ceph2:
|
||||
CephClusterName: ceph2
|
||||
CephClusterFSID: af25554b-42f6-4d2b-9b9b-d08a1132d3e8
|
||||
CinderRbdAvailabilityZone: ceph2-AZ
|
||||
tripleo_ceph3:
|
||||
CephClusterName: ceph3
|
||||
CephClusterFSID: e2cba068-5f14-4b0f-b047-acf375c0004a
|
||||
CinderRbdAvailabilityZone: ceph3-AZ
|
||||
CinderBackupBackend: ceph
|
||||
GlanceBackend: cinder
|
||||
GlanceStoreDescription: 'Cinder glance store'
|
||||
|
|
|
@ -0,0 +1,60 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
CEPH_INFO=($*)
|
||||
|
||||
if [ -z "$CEPH_INFO" ]; then
|
||||
echo "error: At least one CLUSTER:CLIENT tuple must be specified"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "------------------------------------------------"
|
||||
echo "Initializing virsh secrets for: ${CEPH_INFO[@]}"
|
||||
|
||||
for INFO in ${CEPH_INFO[@]}; do
|
||||
IFS=: read CLUSTER CLIENT <<< $INFO
|
||||
FSID=$(awk '$1 == "fsid" {print $3}' /etc/ceph/${CLUSTER}.conf)
|
||||
|
||||
echo "--------"
|
||||
echo "Initializing the virsh secret for '$CLUSTER' cluster ($FSID) '$CLIENT' client"
|
||||
|
||||
# Ensure the secret XML file exists. Puppet should have created a secret.xml
|
||||
# file for the first cluster's secret, so detect when to use that file.
|
||||
if grep -q $FSID /etc/nova/secret.xml; then
|
||||
SECRET_FILE="/etc/nova/secret.xml"
|
||||
SECRET_NAME="client.${CLIENT} secret"
|
||||
else
|
||||
SECRET_FILE="/etc/nova/${CLUSTER}-secret.xml"
|
||||
SECRET_NAME="${CLUSTER}.client.${CLIENT} secret"
|
||||
fi
|
||||
|
||||
if [ ! -f $SECRET_FILE ]; then
|
||||
echo "Creating $SECRET_FILE"
|
||||
cat <<EOF > $SECRET_FILE
|
||||
<secret ephemeral='no' private='no'>
|
||||
<usage type='ceph'>
|
||||
<name>${SECRET_NAME}</name>
|
||||
</usage>
|
||||
<uuid>${FSID}</uuid>
|
||||
</secret>
|
||||
EOF
|
||||
else
|
||||
echo "The $SECRET_FILE file already exists"
|
||||
fi
|
||||
|
||||
# Ensure the libvirt secret is defined
|
||||
if /usr/bin/virsh secret-list | grep -q $FSID; then
|
||||
echo "The virsh secret for $FSID has already been defined"
|
||||
else
|
||||
/usr/bin/virsh secret-define --file $SECRET_FILE
|
||||
fi
|
||||
|
||||
# Fetch the key from the keyring and ensure the secret is set
|
||||
KEY=$(awk '$1 == "key" {print $3}' /etc/ceph/${CLUSTER}.client.${CLIENT}.keyring)
|
||||
if /usr/bin/virsh secret-get-value $FSID 2>/dev/null | grep -q $KEY; then
|
||||
echo "The virsh secret for $FSID has already been set"
|
||||
else
|
||||
/usr/bin/virsh secret-set-value --secret $FSID --base64 $KEY
|
||||
fi
|
||||
done
|
|
@ -87,6 +87,23 @@ parameters:
|
|||
description: >
|
||||
The Ceph cluster name must be at least 1 character and contain only
|
||||
letters and numbers.
|
||||
CinderRbdMultiConfig:
|
||||
type: json
|
||||
default: {}
|
||||
description: |
|
||||
Dictionary of settings when configuring multiple RBD backends. The
|
||||
hash key is the backend name, and the value is a dictionary of parameter
|
||||
values unique to that backend. The following parameters are required,
|
||||
and must match the corresponding value defined in CephExternalMultiConfig.
|
||||
CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
|
||||
CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
|
||||
The following parameters are optional, and override the corresponding
|
||||
parameter's default value.
|
||||
CephClientUserName
|
||||
CinderRbdPoolName
|
||||
CinderRbdExtraPools
|
||||
CinderRbdAvailabilityZone
|
||||
CinderRbdFlattenVolumeFromSnapshot
|
||||
|
||||
conditions:
|
||||
cvol_active_active_tls_enabled:
|
||||
|
@ -158,21 +175,35 @@ outputs:
|
|||
cinder_common_kolla_permissions:
|
||||
description: Common kolla permissions for cinder-volume and cinder-backup services
|
||||
value:
|
||||
- path: /var/log/cinder
|
||||
owner: cinder:cinder
|
||||
recurse: true
|
||||
- path:
|
||||
str_replace:
|
||||
template: /etc/ceph/CLUSTER.client.USER.keyring
|
||||
params:
|
||||
CLUSTER: {get_param: CephClusterName}
|
||||
USER: {get_param: CephClientUserName}
|
||||
owner: cinder:cinder
|
||||
perm: '0600'
|
||||
- path: /etc/pki/tls/certs/etcd.crt
|
||||
owner: cinder:cinder
|
||||
- path: /etc/pki/tls/private/etcd.key
|
||||
owner: cinder:cinder
|
||||
list_concat:
|
||||
-
|
||||
- path: /var/log/cinder
|
||||
owner: cinder:cinder
|
||||
recurse: true
|
||||
- path:
|
||||
str_replace:
|
||||
template: /etc/ceph/CLUSTER.client.USER.keyring
|
||||
params:
|
||||
CLUSTER: {get_param: CephClusterName}
|
||||
USER: {get_param: CephClientUserName}
|
||||
owner: cinder:cinder
|
||||
perm: '0600'
|
||||
- path: /etc/pki/tls/certs/etcd.crt
|
||||
owner: cinder:cinder
|
||||
- path: /etc/pki/tls/private/etcd.key
|
||||
owner: cinder:cinder
|
||||
- repeat:
|
||||
template:
|
||||
path: /etc/ceph/<%keyring%>
|
||||
owner: cinder:cinder
|
||||
perm: '0600'
|
||||
for_each:
|
||||
<%keyring%>:
|
||||
yaql:
|
||||
expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u)))
|
||||
data:
|
||||
default_user: {get_param: CephClientUserName}
|
||||
multiconfig: {get_param: CinderRbdMultiConfig}
|
||||
|
||||
cinder_volume_host_prep_tasks:
|
||||
description: Host prep tasks for the cinder-volume service (HA or non-HA)
|
||||
|
|
|
@ -127,6 +127,10 @@ parameters:
|
|||
Valid values are 'auto', 'true' or 'false'. Effective when
|
||||
CinderEnableNfsBackend is true.
|
||||
type: string
|
||||
CinderRbdBackendName:
|
||||
type: comma_delimited_list
|
||||
default: 'tripleo_ceph'
|
||||
description: A list of Cinder RBD backend names.
|
||||
CinderRbdAvailabilityZone:
|
||||
default: ''
|
||||
description: >
|
||||
|
@ -144,13 +148,29 @@ parameters:
|
|||
list. This is in addition to the standard RBD backend driver
|
||||
associated with the CinderRbdPoolName.
|
||||
type: comma_delimited_list
|
||||
|
||||
CinderRbdFlattenVolumeFromSnapshot:
|
||||
default: false
|
||||
description: >
|
||||
Whether RBD volumes created from a snapshot should be flattened
|
||||
in order to remove a dependency on the snapshot.
|
||||
type: boolean
|
||||
CinderRbdMultiConfig:
|
||||
type: json
|
||||
default: {}
|
||||
description: |
|
||||
Dictionary of settings when configuring multiple RBD backends. The
|
||||
hash key is the backend name, and the value is a dictionary of parameter
|
||||
values unique to that backend. The following parameters are required,
|
||||
and must match the corresponding value defined in CephExternalMultiConfig.
|
||||
CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
|
||||
CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
|
||||
The following parameters are optional, and override the corresponding
|
||||
parameter's default value.
|
||||
CephClientUserName
|
||||
CinderRbdPoolName
|
||||
CinderRbdExtraPools
|
||||
CinderRbdAvailabilityZone
|
||||
CinderRbdFlattenVolumeFromSnapshot
|
||||
CephClusterFSID:
|
||||
type: string
|
||||
description: The Ceph cluster FSID. Must be a UUID.
|
||||
|
@ -229,6 +249,7 @@ outputs:
|
|||
tripleo::profile::base::cinder::volume::nfs::cinder_nas_secure_file_permissions: {get_param: CinderNasSecureFilePermissions}
|
||||
tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_helper: {get_param: CinderISCSIHelper}
|
||||
tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_protocol: {get_param: CinderISCSIProtocol}
|
||||
tripleo::profile::base::cinder::volume::rbd::backend_name: {get_param: CinderRbdBackendName}
|
||||
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_ceph_conf:
|
||||
list_join:
|
||||
- ''
|
||||
|
@ -239,8 +260,8 @@ outputs:
|
|||
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools: {get_param: CinderRbdExtraPools}
|
||||
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_secret_uuid: {get_param: CephClusterFSID}
|
||||
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: {get_param: CephClientUserName}
|
||||
tripleo::profile::base::cinder::volume::cinder_rbd_ceph_conf_path: {get_param: CephConfigPath}
|
||||
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_flatten_volume_from_snapshot: {get_param: CinderRbdFlattenVolumeFromSnapshot}
|
||||
tripleo::profile::base::cinder::volume::rbd::multi_config: {get_param: CinderRbdMultiConfig}
|
||||
# NOTE: bind IP is found in hiera replacing the network name with the local node IP
|
||||
# for the given network; replacement examples (eg. for internal_api):
|
||||
# internal_api -> IP
|
||||
|
|
|
@ -59,6 +59,23 @@ parameters:
|
|||
default: "/var/lib/tripleo-config/ceph"
|
||||
description: |
|
||||
The path where the Ceph Cluster config files are stored on the host.
|
||||
CinderRbdMultiConfig:
|
||||
type: json
|
||||
default: {}
|
||||
description: |
|
||||
Dictionary of settings when configuring multiple RBD backends. The
|
||||
hash key is the backend name, and the value is a dictionary of parameter
|
||||
values unique to that backend. The following parameters are required,
|
||||
and must match the corresponding value defined in CephExternalMultiConfig.
|
||||
CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
|
||||
CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
|
||||
The following parameters are optional, and override the corresponding
|
||||
parameter's default value.
|
||||
CephClientUserName
|
||||
CinderRbdPoolName
|
||||
CinderRbdExtraPools
|
||||
CinderRbdAvailabilityZone
|
||||
CinderRbdFlattenVolumeFromSnapshot
|
||||
NovaComputeOptVolumes:
|
||||
default: []
|
||||
description: list of optional volumes to be mounted
|
||||
|
@ -1269,6 +1286,18 @@ outputs:
|
|||
data:
|
||||
user: {get_param: CephClientUserName}
|
||||
multistore: {get_param: GlanceMultistoreConfig}
|
||||
- repeat:
|
||||
template:
|
||||
path: /etc/ceph/<%keyring%>
|
||||
owner: nova:nova
|
||||
perm: '0600'
|
||||
for_each:
|
||||
<%keyring%>:
|
||||
yaql:
|
||||
expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u)))
|
||||
data:
|
||||
default_user: {get_param: CephClientUserName}
|
||||
multiconfig: {get_param: CinderRbdMultiConfig}
|
||||
container_config_scripts:
|
||||
map_merge:
|
||||
- {get_attr: [ContainersCommon, container_config_scripts]}
|
||||
|
|
|
@ -84,6 +84,23 @@ parameters:
|
|||
description: >
|
||||
The Ceph cluster name must be at least 1 character and contain only
|
||||
letters and numbers.
|
||||
CinderRbdMultiConfig:
|
||||
type: json
|
||||
default: {}
|
||||
description: |
|
||||
Dictionary of settings when configuring multiple RBD backends. The
|
||||
hash key is the backend name, and the value is a dictionary of parameter
|
||||
values unique to that backend. The following parameters are required,
|
||||
and must match the corresponding value defined in CephExternalMultiConfig.
|
||||
CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
|
||||
CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
|
||||
The following parameters are optional, and override the corresponding
|
||||
parameter's default value.
|
||||
CephClientUserName
|
||||
CinderRbdPoolName
|
||||
CinderRbdExtraPools
|
||||
CinderRbdAvailabilityZone
|
||||
CinderRbdFlattenVolumeFromSnapshot
|
||||
UseTLSTransportForVnc:
|
||||
type: boolean
|
||||
default: true
|
||||
|
@ -505,6 +522,18 @@ outputs:
|
|||
USER: {get_param: CephClientUserName}
|
||||
owner: nova:nova
|
||||
perm: '0600'
|
||||
- repeat:
|
||||
template:
|
||||
path: /etc/ceph/<%keyring%>
|
||||
owner: nova:nova
|
||||
perm: '0600'
|
||||
for_each:
|
||||
<%keyring%>:
|
||||
yaql:
|
||||
expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u)))
|
||||
data:
|
||||
default_user: {get_param: CephClientUserName}
|
||||
multiconfig: {get_param: CinderRbdMultiConfig}
|
||||
/var/lib/kolla/config_files/nova_virtlogd.json:
|
||||
command: /usr/sbin/virtlogd --config /etc/libvirt/virtlogd.conf
|
||||
config_files:
|
||||
|
@ -530,6 +559,9 @@ outputs:
|
|||
- use_tls_for_live_migration
|
||||
- '--listen'
|
||||
- ''
|
||||
nova_libvirt_init_secret.sh:
|
||||
mode: "0755"
|
||||
content: { get_file: ../../container_config_scripts/nova_libvirt_init_secret.sh }
|
||||
docker_config:
|
||||
step_3:
|
||||
nova_virtlogd:
|
||||
|
@ -671,18 +703,36 @@ outputs:
|
|||
list_concat:
|
||||
- {get_attr: [ContainersCommon, volumes]}
|
||||
-
|
||||
- /var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova:ro
|
||||
- /var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova
|
||||
- /etc/libvirt:/etc/libvirt
|
||||
- /run/libvirt:/run/libvirt:shared
|
||||
- /var/lib/libvirt:/var/lib/libvirt:shared
|
||||
- /var/lib/container-config-scripts/nova_libvirt_init_secret.sh:/nova_libvirt_init_secret.sh:ro
|
||||
-
|
||||
str_replace:
|
||||
template: HOST_CEPH:/etc/ceph:ro
|
||||
params:
|
||||
HOST_CEPH: {get_param: CephConfigPath}
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- str_replace:
|
||||
template: /usr/bin/virsh secret-define --file /etc/nova/secret.xml && /usr/bin/virsh secret-set-value --secret 'SECRET_UUID' --base64 'SECRET_KEY'
|
||||
params:
|
||||
SECRET_UUID: {get_param: CephClusterFSID}
|
||||
SECRET_KEY: {get_param: CephClientKey}
|
||||
list_join:
|
||||
- ' '
|
||||
- - str_replace:
|
||||
template:
|
||||
"/nova_libvirt_init_secret.sh CLUSTER:USER"
|
||||
params:
|
||||
CLUSTER: {get_param: CephClusterName}
|
||||
USER: {get_param: CephClientUserName}
|
||||
- repeat:
|
||||
template:
|
||||
<%ceph_info%>
|
||||
for_each:
|
||||
<%ceph_info%>:
|
||||
yaql:
|
||||
expression:
|
||||
let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}:{1}".format($.CephClusterName, $.get("CephClientUserName", $u)))
|
||||
data:
|
||||
default_user: {get_param: CephClientUserName}
|
||||
multiconfig: {get_param: CinderRbdMultiConfig}
|
||||
- {}
|
||||
deploy_steps_tasks:
|
||||
list_concat:
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
features:
|
||||
- |
|
||||
A new ``CinderRbdMultiConfig`` parameter may be used to configure
|
||||
additional cinder RBD backends on external Ceph clusters defined by the
|
||||
``CephExternalMultiConfig`` parameter.
|
Loading…
Reference in New Issue