Merge "Add cinder RBD support for multiple ceph clusters"
This commit is contained in:
commit
6f1305b0c8
|
@ -109,6 +109,16 @@ parameter_defaults:
|
||||||
dashboard_enabled: false
|
dashboard_enabled: false
|
||||||
NovaEnableRbdBackend: true
|
NovaEnableRbdBackend: true
|
||||||
CinderEnableRbdBackend: true
|
CinderEnableRbdBackend: true
|
||||||
|
CinderRbdBackendName: tripleo_ceph,tripleo_ceph2,tripleo_ceph3
|
||||||
|
CinderRbdMultiConfig:
|
||||||
|
tripleo_ceph2:
|
||||||
|
CephClusterName: ceph2
|
||||||
|
CephClusterFSID: af25554b-42f6-4d2b-9b9b-d08a1132d3e8
|
||||||
|
CinderRbdAvailabilityZone: ceph2-AZ
|
||||||
|
tripleo_ceph3:
|
||||||
|
CephClusterName: ceph3
|
||||||
|
CephClusterFSID: e2cba068-5f14-4b0f-b047-acf375c0004a
|
||||||
|
CinderRbdAvailabilityZone: ceph3-AZ
|
||||||
CinderBackupBackend: ceph
|
CinderBackupBackend: ceph
|
||||||
GlanceBackend: cinder
|
GlanceBackend: cinder
|
||||||
GlanceStoreDescription: 'Cinder glance store'
|
GlanceStoreDescription: 'Cinder glance store'
|
||||||
|
|
|
@ -0,0 +1,60 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
CEPH_INFO=($*)
|
||||||
|
|
||||||
|
if [ -z "$CEPH_INFO" ]; then
|
||||||
|
echo "error: At least one CLUSTER:CLIENT tuple must be specified"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "------------------------------------------------"
|
||||||
|
echo "Initializing virsh secrets for: ${CEPH_INFO[@]}"
|
||||||
|
|
||||||
|
for INFO in ${CEPH_INFO[@]}; do
|
||||||
|
IFS=: read CLUSTER CLIENT <<< $INFO
|
||||||
|
FSID=$(awk '$1 == "fsid" {print $3}' /etc/ceph/${CLUSTER}.conf)
|
||||||
|
|
||||||
|
echo "--------"
|
||||||
|
echo "Initializing the virsh secret for '$CLUSTER' cluster ($FSID) '$CLIENT' client"
|
||||||
|
|
||||||
|
# Ensure the secret XML file exists. Puppet should have created a secret.xml
|
||||||
|
# file for the first cluster's secret, so detect when to use that file.
|
||||||
|
if grep -q $FSID /etc/nova/secret.xml; then
|
||||||
|
SECRET_FILE="/etc/nova/secret.xml"
|
||||||
|
SECRET_NAME="client.${CLIENT} secret"
|
||||||
|
else
|
||||||
|
SECRET_FILE="/etc/nova/${CLUSTER}-secret.xml"
|
||||||
|
SECRET_NAME="${CLUSTER}.client.${CLIENT} secret"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f $SECRET_FILE ]; then
|
||||||
|
echo "Creating $SECRET_FILE"
|
||||||
|
cat <<EOF > $SECRET_FILE
|
||||||
|
<secret ephemeral='no' private='no'>
|
||||||
|
<usage type='ceph'>
|
||||||
|
<name>${SECRET_NAME}</name>
|
||||||
|
</usage>
|
||||||
|
<uuid>${FSID}</uuid>
|
||||||
|
</secret>
|
||||||
|
EOF
|
||||||
|
else
|
||||||
|
echo "The $SECRET_FILE file already exists"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Ensure the libvirt secret is defined
|
||||||
|
if /usr/bin/virsh secret-list | grep -q $FSID; then
|
||||||
|
echo "The virsh secret for $FSID has already been defined"
|
||||||
|
else
|
||||||
|
/usr/bin/virsh secret-define --file $SECRET_FILE
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Fetch the key from the keyring and ensure the secret is set
|
||||||
|
KEY=$(awk '$1 == "key" {print $3}' /etc/ceph/${CLUSTER}.client.${CLIENT}.keyring)
|
||||||
|
if /usr/bin/virsh secret-get-value $FSID 2>/dev/null | grep -q $KEY; then
|
||||||
|
echo "The virsh secret for $FSID has already been set"
|
||||||
|
else
|
||||||
|
/usr/bin/virsh secret-set-value --secret $FSID --base64 $KEY
|
||||||
|
fi
|
||||||
|
done
|
|
@ -87,6 +87,23 @@ parameters:
|
||||||
description: >
|
description: >
|
||||||
The Ceph cluster name must be at least 1 character and contain only
|
The Ceph cluster name must be at least 1 character and contain only
|
||||||
letters and numbers.
|
letters and numbers.
|
||||||
|
CinderRbdMultiConfig:
|
||||||
|
type: json
|
||||||
|
default: {}
|
||||||
|
description: |
|
||||||
|
Dictionary of settings when configuring multiple RBD backends. The
|
||||||
|
hash key is the backend name, and the value is a dictionary of parameter
|
||||||
|
values unique to that backend. The following parameters are required,
|
||||||
|
and must match the corresponding value defined in CephExternalMultiConfig.
|
||||||
|
CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
|
||||||
|
CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
|
||||||
|
The following parameters are optional, and override the corresponding
|
||||||
|
parameter's default value.
|
||||||
|
CephClientUserName
|
||||||
|
CinderRbdPoolName
|
||||||
|
CinderRbdExtraPools
|
||||||
|
CinderRbdAvailabilityZone
|
||||||
|
CinderRbdFlattenVolumeFromSnapshot
|
||||||
|
|
||||||
conditions:
|
conditions:
|
||||||
cvol_active_active_tls_enabled:
|
cvol_active_active_tls_enabled:
|
||||||
|
@ -158,21 +175,35 @@ outputs:
|
||||||
cinder_common_kolla_permissions:
|
cinder_common_kolla_permissions:
|
||||||
description: Common kolla permissions for cinder-volume and cinder-backup services
|
description: Common kolla permissions for cinder-volume and cinder-backup services
|
||||||
value:
|
value:
|
||||||
- path: /var/log/cinder
|
list_concat:
|
||||||
owner: cinder:cinder
|
-
|
||||||
recurse: true
|
- path: /var/log/cinder
|
||||||
- path:
|
owner: cinder:cinder
|
||||||
str_replace:
|
recurse: true
|
||||||
template: /etc/ceph/CLUSTER.client.USER.keyring
|
- path:
|
||||||
params:
|
str_replace:
|
||||||
CLUSTER: {get_param: CephClusterName}
|
template: /etc/ceph/CLUSTER.client.USER.keyring
|
||||||
USER: {get_param: CephClientUserName}
|
params:
|
||||||
owner: cinder:cinder
|
CLUSTER: {get_param: CephClusterName}
|
||||||
perm: '0600'
|
USER: {get_param: CephClientUserName}
|
||||||
- path: /etc/pki/tls/certs/etcd.crt
|
owner: cinder:cinder
|
||||||
owner: cinder:cinder
|
perm: '0600'
|
||||||
- path: /etc/pki/tls/private/etcd.key
|
- path: /etc/pki/tls/certs/etcd.crt
|
||||||
owner: cinder:cinder
|
owner: cinder:cinder
|
||||||
|
- path: /etc/pki/tls/private/etcd.key
|
||||||
|
owner: cinder:cinder
|
||||||
|
- repeat:
|
||||||
|
template:
|
||||||
|
path: /etc/ceph/<%keyring%>
|
||||||
|
owner: cinder:cinder
|
||||||
|
perm: '0600'
|
||||||
|
for_each:
|
||||||
|
<%keyring%>:
|
||||||
|
yaql:
|
||||||
|
expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u)))
|
||||||
|
data:
|
||||||
|
default_user: {get_param: CephClientUserName}
|
||||||
|
multiconfig: {get_param: CinderRbdMultiConfig}
|
||||||
|
|
||||||
cinder_volume_host_prep_tasks:
|
cinder_volume_host_prep_tasks:
|
||||||
description: Host prep tasks for the cinder-volume service (HA or non-HA)
|
description: Host prep tasks for the cinder-volume service (HA or non-HA)
|
||||||
|
|
|
@ -127,6 +127,10 @@ parameters:
|
||||||
Valid values are 'auto', 'true' or 'false'. Effective when
|
Valid values are 'auto', 'true' or 'false'. Effective when
|
||||||
CinderEnableNfsBackend is true.
|
CinderEnableNfsBackend is true.
|
||||||
type: string
|
type: string
|
||||||
|
CinderRbdBackendName:
|
||||||
|
type: comma_delimited_list
|
||||||
|
default: 'tripleo_ceph'
|
||||||
|
description: A list of Cinder RBD backend names.
|
||||||
CinderRbdAvailabilityZone:
|
CinderRbdAvailabilityZone:
|
||||||
default: ''
|
default: ''
|
||||||
description: >
|
description: >
|
||||||
|
@ -144,13 +148,29 @@ parameters:
|
||||||
list. This is in addition to the standard RBD backend driver
|
list. This is in addition to the standard RBD backend driver
|
||||||
associated with the CinderRbdPoolName.
|
associated with the CinderRbdPoolName.
|
||||||
type: comma_delimited_list
|
type: comma_delimited_list
|
||||||
|
|
||||||
CinderRbdFlattenVolumeFromSnapshot:
|
CinderRbdFlattenVolumeFromSnapshot:
|
||||||
default: false
|
default: false
|
||||||
description: >
|
description: >
|
||||||
Whether RBD volumes created from a snapshot should be flattened
|
Whether RBD volumes created from a snapshot should be flattened
|
||||||
in order to remove a dependency on the snapshot.
|
in order to remove a dependency on the snapshot.
|
||||||
type: boolean
|
type: boolean
|
||||||
|
CinderRbdMultiConfig:
|
||||||
|
type: json
|
||||||
|
default: {}
|
||||||
|
description: |
|
||||||
|
Dictionary of settings when configuring multiple RBD backends. The
|
||||||
|
hash key is the backend name, and the value is a dictionary of parameter
|
||||||
|
values unique to that backend. The following parameters are required,
|
||||||
|
and must match the corresponding value defined in CephExternalMultiConfig.
|
||||||
|
CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
|
||||||
|
CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
|
||||||
|
The following parameters are optional, and override the corresponding
|
||||||
|
parameter's default value.
|
||||||
|
CephClientUserName
|
||||||
|
CinderRbdPoolName
|
||||||
|
CinderRbdExtraPools
|
||||||
|
CinderRbdAvailabilityZone
|
||||||
|
CinderRbdFlattenVolumeFromSnapshot
|
||||||
CephClusterFSID:
|
CephClusterFSID:
|
||||||
type: string
|
type: string
|
||||||
description: The Ceph cluster FSID. Must be a UUID.
|
description: The Ceph cluster FSID. Must be a UUID.
|
||||||
|
@ -229,6 +249,7 @@ outputs:
|
||||||
tripleo::profile::base::cinder::volume::nfs::cinder_nas_secure_file_permissions: {get_param: CinderNasSecureFilePermissions}
|
tripleo::profile::base::cinder::volume::nfs::cinder_nas_secure_file_permissions: {get_param: CinderNasSecureFilePermissions}
|
||||||
tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_helper: {get_param: CinderISCSIHelper}
|
tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_helper: {get_param: CinderISCSIHelper}
|
||||||
tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_protocol: {get_param: CinderISCSIProtocol}
|
tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_protocol: {get_param: CinderISCSIProtocol}
|
||||||
|
tripleo::profile::base::cinder::volume::rbd::backend_name: {get_param: CinderRbdBackendName}
|
||||||
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_ceph_conf:
|
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_ceph_conf:
|
||||||
list_join:
|
list_join:
|
||||||
- ''
|
- ''
|
||||||
|
@ -239,8 +260,8 @@ outputs:
|
||||||
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools: {get_param: CinderRbdExtraPools}
|
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools: {get_param: CinderRbdExtraPools}
|
||||||
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_secret_uuid: {get_param: CephClusterFSID}
|
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_secret_uuid: {get_param: CephClusterFSID}
|
||||||
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: {get_param: CephClientUserName}
|
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: {get_param: CephClientUserName}
|
||||||
tripleo::profile::base::cinder::volume::cinder_rbd_ceph_conf_path: {get_param: CephConfigPath}
|
|
||||||
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_flatten_volume_from_snapshot: {get_param: CinderRbdFlattenVolumeFromSnapshot}
|
tripleo::profile::base::cinder::volume::rbd::cinder_rbd_flatten_volume_from_snapshot: {get_param: CinderRbdFlattenVolumeFromSnapshot}
|
||||||
|
tripleo::profile::base::cinder::volume::rbd::multi_config: {get_param: CinderRbdMultiConfig}
|
||||||
# NOTE: bind IP is found in hiera replacing the network name with the local node IP
|
# NOTE: bind IP is found in hiera replacing the network name with the local node IP
|
||||||
# for the given network; replacement examples (eg. for internal_api):
|
# for the given network; replacement examples (eg. for internal_api):
|
||||||
# internal_api -> IP
|
# internal_api -> IP
|
||||||
|
|
|
@ -59,6 +59,23 @@ parameters:
|
||||||
default: "/var/lib/tripleo-config/ceph"
|
default: "/var/lib/tripleo-config/ceph"
|
||||||
description: |
|
description: |
|
||||||
The path where the Ceph Cluster config files are stored on the host.
|
The path where the Ceph Cluster config files are stored on the host.
|
||||||
|
CinderRbdMultiConfig:
|
||||||
|
type: json
|
||||||
|
default: {}
|
||||||
|
description: |
|
||||||
|
Dictionary of settings when configuring multiple RBD backends. The
|
||||||
|
hash key is the backend name, and the value is a dictionary of parameter
|
||||||
|
values unique to that backend. The following parameters are required,
|
||||||
|
and must match the corresponding value defined in CephExternalMultiConfig.
|
||||||
|
CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
|
||||||
|
CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
|
||||||
|
The following parameters are optional, and override the corresponding
|
||||||
|
parameter's default value.
|
||||||
|
CephClientUserName
|
||||||
|
CinderRbdPoolName
|
||||||
|
CinderRbdExtraPools
|
||||||
|
CinderRbdAvailabilityZone
|
||||||
|
CinderRbdFlattenVolumeFromSnapshot
|
||||||
NovaComputeOptVolumes:
|
NovaComputeOptVolumes:
|
||||||
default: []
|
default: []
|
||||||
description: list of optional volumes to be mounted
|
description: list of optional volumes to be mounted
|
||||||
|
@ -1279,6 +1296,18 @@ outputs:
|
||||||
data:
|
data:
|
||||||
user: {get_param: CephClientUserName}
|
user: {get_param: CephClientUserName}
|
||||||
multistore: {get_param: GlanceMultistoreConfig}
|
multistore: {get_param: GlanceMultistoreConfig}
|
||||||
|
- repeat:
|
||||||
|
template:
|
||||||
|
path: /etc/ceph/<%keyring%>
|
||||||
|
owner: nova:nova
|
||||||
|
perm: '0600'
|
||||||
|
for_each:
|
||||||
|
<%keyring%>:
|
||||||
|
yaql:
|
||||||
|
expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u)))
|
||||||
|
data:
|
||||||
|
default_user: {get_param: CephClientUserName}
|
||||||
|
multiconfig: {get_param: CinderRbdMultiConfig}
|
||||||
container_config_scripts:
|
container_config_scripts:
|
||||||
map_merge:
|
map_merge:
|
||||||
- {get_attr: [ContainersCommon, container_config_scripts]}
|
- {get_attr: [ContainersCommon, container_config_scripts]}
|
||||||
|
|
|
@ -84,6 +84,23 @@ parameters:
|
||||||
description: >
|
description: >
|
||||||
The Ceph cluster name must be at least 1 character and contain only
|
The Ceph cluster name must be at least 1 character and contain only
|
||||||
letters and numbers.
|
letters and numbers.
|
||||||
|
CinderRbdMultiConfig:
|
||||||
|
type: json
|
||||||
|
default: {}
|
||||||
|
description: |
|
||||||
|
Dictionary of settings when configuring multiple RBD backends. The
|
||||||
|
hash key is the backend name, and the value is a dictionary of parameter
|
||||||
|
values unique to that backend. The following parameters are required,
|
||||||
|
and must match the corresponding value defined in CephExternalMultiConfig.
|
||||||
|
CephClusterName (must match the CephExternalMultiConfig entry's 'cluster')
|
||||||
|
CephClusterFSID (must match the CephExternalMultiConfig entry's 'fsid')
|
||||||
|
The following parameters are optional, and override the corresponding
|
||||||
|
parameter's default value.
|
||||||
|
CephClientUserName
|
||||||
|
CinderRbdPoolName
|
||||||
|
CinderRbdExtraPools
|
||||||
|
CinderRbdAvailabilityZone
|
||||||
|
CinderRbdFlattenVolumeFromSnapshot
|
||||||
UseTLSTransportForVnc:
|
UseTLSTransportForVnc:
|
||||||
type: boolean
|
type: boolean
|
||||||
default: true
|
default: true
|
||||||
|
@ -505,6 +522,18 @@ outputs:
|
||||||
USER: {get_param: CephClientUserName}
|
USER: {get_param: CephClientUserName}
|
||||||
owner: nova:nova
|
owner: nova:nova
|
||||||
perm: '0600'
|
perm: '0600'
|
||||||
|
- repeat:
|
||||||
|
template:
|
||||||
|
path: /etc/ceph/<%keyring%>
|
||||||
|
owner: nova:nova
|
||||||
|
perm: '0600'
|
||||||
|
for_each:
|
||||||
|
<%keyring%>:
|
||||||
|
yaql:
|
||||||
|
expression: let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u)))
|
||||||
|
data:
|
||||||
|
default_user: {get_param: CephClientUserName}
|
||||||
|
multiconfig: {get_param: CinderRbdMultiConfig}
|
||||||
/var/lib/kolla/config_files/nova_virtlogd.json:
|
/var/lib/kolla/config_files/nova_virtlogd.json:
|
||||||
command: /usr/sbin/virtlogd --config /etc/libvirt/virtlogd.conf
|
command: /usr/sbin/virtlogd --config /etc/libvirt/virtlogd.conf
|
||||||
config_files:
|
config_files:
|
||||||
|
@ -530,6 +559,9 @@ outputs:
|
||||||
- use_tls_for_live_migration
|
- use_tls_for_live_migration
|
||||||
- '--listen'
|
- '--listen'
|
||||||
- ''
|
- ''
|
||||||
|
nova_libvirt_init_secret.sh:
|
||||||
|
mode: "0755"
|
||||||
|
content: { get_file: ../../container_config_scripts/nova_libvirt_init_secret.sh }
|
||||||
docker_config:
|
docker_config:
|
||||||
step_3:
|
step_3:
|
||||||
nova_virtlogd:
|
nova_virtlogd:
|
||||||
|
@ -671,18 +703,36 @@ outputs:
|
||||||
list_concat:
|
list_concat:
|
||||||
- {get_attr: [ContainersCommon, volumes]}
|
- {get_attr: [ContainersCommon, volumes]}
|
||||||
-
|
-
|
||||||
- /var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova:ro
|
- /var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova
|
||||||
- /etc/libvirt:/etc/libvirt
|
- /etc/libvirt:/etc/libvirt
|
||||||
- /run/libvirt:/run/libvirt:shared
|
- /run/libvirt:/run/libvirt:shared
|
||||||
- /var/lib/libvirt:/var/lib/libvirt:shared
|
- /var/lib/libvirt:/var/lib/libvirt:shared
|
||||||
|
- /var/lib/container-config-scripts/nova_libvirt_init_secret.sh:/nova_libvirt_init_secret.sh:ro
|
||||||
|
-
|
||||||
|
str_replace:
|
||||||
|
template: HOST_CEPH:/etc/ceph:ro
|
||||||
|
params:
|
||||||
|
HOST_CEPH: {get_param: CephConfigPath}
|
||||||
command:
|
command:
|
||||||
- /bin/bash
|
list_join:
|
||||||
- -c
|
- ' '
|
||||||
- str_replace:
|
- - str_replace:
|
||||||
template: /usr/bin/virsh secret-define --file /etc/nova/secret.xml && /usr/bin/virsh secret-set-value --secret 'SECRET_UUID' --base64 'SECRET_KEY'
|
template:
|
||||||
params:
|
"/nova_libvirt_init_secret.sh CLUSTER:USER"
|
||||||
SECRET_UUID: {get_param: CephClusterFSID}
|
params:
|
||||||
SECRET_KEY: {get_param: CephClientKey}
|
CLUSTER: {get_param: CephClusterName}
|
||||||
|
USER: {get_param: CephClientUserName}
|
||||||
|
- repeat:
|
||||||
|
template:
|
||||||
|
<%ceph_info%>
|
||||||
|
for_each:
|
||||||
|
<%ceph_info%>:
|
||||||
|
yaql:
|
||||||
|
expression:
|
||||||
|
let(u => $.data.default_user) -> $.data.multiconfig.values().select("{0}:{1}".format($.CephClusterName, $.get("CephClientUserName", $u)))
|
||||||
|
data:
|
||||||
|
default_user: {get_param: CephClientUserName}
|
||||||
|
multiconfig: {get_param: CinderRbdMultiConfig}
|
||||||
- {}
|
- {}
|
||||||
deploy_steps_tasks:
|
deploy_steps_tasks:
|
||||||
list_concat:
|
list_concat:
|
||||||
|
|
|
@ -0,0 +1,6 @@
|
||||||
|
---
|
||||||
|
features:
|
||||||
|
- |
|
||||||
|
A new ``CinderRbdMultiConfig`` parameter may be used to configure
|
||||||
|
additional cinder RBD backends on external Ceph clusters defined by the
|
||||||
|
``CephExternalMultiConfig`` parameter.
|
Loading…
Reference in New Issue