From 22a5dd19bd3e4404cd5419972c437b18391f7f0c Mon Sep 17 00:00:00 2001 From: Alan Bishop Date: Thu, 1 Apr 2021 11:50:27 -0700 Subject: [PATCH] Add cinder RBD support for multiple ceph clusters Following the pattern of other cinder backends, a new tripleo::profile::base::cinder::volume::rbd::multi_config parameter adds support for configuring cinder RBD backends associated with multiple ceph clusters. The multi_config parameter is a hash that specifies the settings required to access each additional cluster (FSID, pool and client names, etc.). This patch also deprecates legacy code that managed ACLs for the ceph client keyring. THT already ensures cinder can access the keyring using kolla_config permissions. Change-Id: I040e25341c9869ad289d7e7c98e831caef23fece --- manifests/profile/base/cinder/volume.pp | 58 +++++++++-------- manifests/profile/base/cinder/volume/rbd.pp | 63 +++++++++++++------ ...nder-rbd-multiconfig-285d1542ef08fa10.yaml | 6 ++ ...leo_profile_base_cinder_volume_rbd_spec.rb | 60 ++++++++++++++++++ ...tripleo_profile_base_cinder_volume_spec.rb | 14 ++--- spec/fixtures/hieradata/step5.yaml | 7 +++ 6 files changed, 153 insertions(+), 55 deletions(-) create mode 100644 releasenotes/notes/cinder-rbd-multiconfig-285d1542ef08fa10.yaml diff --git a/manifests/profile/base/cinder/volume.pp b/manifests/profile/base/cinder/volume.pp index 68c8ffb51..8f04ffe58 100644 --- a/manifests/profile/base/cinder/volume.pp +++ b/manifests/profile/base/cinder/volume.pp @@ -102,14 +102,6 @@ # (Optional) List of additional backend stanzas to activate # Defaults to hiera('cinder_user_enabled_backends') # -# [*cinder_rbd_client_name*] -# (Optional) Name of RBD client -# Defaults to hiera('tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name') -# -# [*cinder_rbd_ceph_conf_path*] -# (Optional) The path where the Ceph Cluster config files are stored on the host -# Defaults to '/etc/ceph' -# # [*cinder_volume_cluster*] # (Optional) Name of the cluster when running in active-active mode # Defaults to '' @@ -139,6 +131,16 @@ # for more details. # Defaults to hiera('step') # +# DEPRECATED PARAMETERS +# +# [*cinder_rbd_client_name*] +# (Optional) Name of RBD client +# Defaults to undef +# +# [*cinder_rbd_ceph_conf_path*] +# (Optional) The path where the Ceph Cluster config files are stored on the host +# Defaults to undef +# class tripleo::profile::base::cinder::volume ( $cinder_enable_pure_backend = false, $cinder_enable_dellsc_backend = false, @@ -161,8 +163,6 @@ class tripleo::profile::base::cinder::volume ( $cinder_enable_vrts_hs_backend = false, $cinder_enable_nvmeof_backend = false, $cinder_user_enabled_backends = hiera('cinder_user_enabled_backends', undef), - $cinder_rbd_ceph_conf_path = '/etc/ceph', - $cinder_rbd_client_name = hiera('tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name','openstack'), $cinder_volume_cluster = '', $enable_internal_tls = hiera('enable_internal_tls', false), $etcd_certificate_specs = hiera('tripleo::profile::base::etcd::certificate_specs', {}), @@ -170,6 +170,9 @@ class tripleo::profile::base::cinder::volume ( $etcd_host = hiera('etcd_vip', undef), $etcd_port = hiera('tripleo::profile::base::etcd::client_port', '2379'), $step = Integer(hiera('step')), + # DEPRECATED PARAMETERS + $cinder_rbd_ceph_conf_path = undef, + $cinder_rbd_client_name = undef, ) { include tripleo::profile::base::cinder @@ -326,26 +329,27 @@ class tripleo::profile::base::cinder::volume ( if $cinder_enable_rbd_backend { include tripleo::profile::base::cinder::volume::rbd - $cinder_rbd_backend_name = hiera('cinder::backend::rbd::volume_backend_name', 'tripleo_ceph') + $cinder_rbd_backend_name = hiera('tripleo::profile::base::cinder::volume::rbd::backend_name', + ['tripleo_ceph']) - exec{ "exec-setfacl-${cinder_rbd_client_name}-cinder": - path => ['/bin', '/usr/bin'], - command => "setfacl -m u:cinder:r-- ${cinder_rbd_ceph_conf_path}/ceph.client.${cinder_rbd_client_name}.keyring", - unless => "getfacl ${cinder_rbd_ceph_conf_path}/ceph.client.${cinder_rbd_client_name}.keyring | grep -q user:cinder:r--", - } - -> exec{ "exec-setfacl-${cinder_rbd_client_name}-cinder-mask": - path => ['/bin', '/usr/bin'], - command => "setfacl -m m::r ${cinder_rbd_ceph_conf_path}/ceph.client.${cinder_rbd_client_name}.keyring", - unless => "getfacl ${cinder_rbd_ceph_conf_path}/ceph.client.${cinder_rbd_client_name}.keyring | grep -q mask::r", - } - - $cinder_rbd_extra_pools = hiera('tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools', undef) - if $cinder_rbd_extra_pools { - $base_name = $cinder_rbd_backend_name - $cinder_rbd_extra_backend_names = $cinder_rbd_extra_pools.map |$pool_name| { "${base_name}_${pool_name}" } + $extra_pools = hiera('tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools', undef) + if empty($extra_pools) { + $extra_backend_names = [] } else { - $cinder_rbd_extra_backend_names = undef + # These $extra_pools are associated with the first backend + $base_name = any2array($cinder_rbd_backend_name)[0] + $extra_backend_names = any2array($extra_pools).map |$pool_name| { "${base_name}_${pool_name}" } } + + # Each $multi_config backend can specify its own list of extra pools. The + # backend names are the $multi_config hash keys. + $multi_config = hiera('tripleo::profile::base::cinder::volume::rbd::multi_config', {}) + $extra_multiconfig_backend_names = $multi_config.map |$base_name, $backend_config| { + $backend_extra_pools = $backend_config['CinderRbdExtraPools'] + any2array($backend_extra_pools).map |$pool_name| { "${base_name}_${pool_name}" } + } + + $cinder_rbd_extra_backend_names = flatten($extra_backend_names, $extra_multiconfig_backend_names) } else { $cinder_rbd_backend_name = undef $cinder_rbd_extra_backend_names = undef diff --git a/manifests/profile/base/cinder/volume/rbd.pp b/manifests/profile/base/cinder/volume/rbd.pp index 0db22a886..4954098c5 100644 --- a/manifests/profile/base/cinder/volume/rbd.pp +++ b/manifests/profile/base/cinder/volume/rbd.pp @@ -19,8 +19,8 @@ # === Parameters # # [*backend_name*] -# (Optional) Name given to the Cinder backend stanza -# Defaults to 'tripleo_ceph' +# (Optional) List of names given to the Cinder backend stanza. +# Defaults to hiera('cinder::backend::rbd::volume_backend_name', ['tripleo_ceph']) # # [*backend_availability_zone*] # (Optional) Availability zone for this volume backend @@ -56,13 +56,17 @@ # in order to remove a dependency on the snapshot. # Defaults to hiera('cinder::backend::rbd::flatten_volume_from_snapshot, undef) # +# [*multi_config*] +# (Optional) A config hash when multiple backends are used. +# Defaults to {} +# # [*step*] # (Optional) The current step in deployment. See tripleo-heat-templates # for more details. # Defaults to hiera('step') # class tripleo::profile::base::cinder::volume::rbd ( - $backend_name = hiera('cinder::backend::rbd::volume_backend_name', 'tripleo_ceph'), + $backend_name = hiera('cinder::backend::rbd::volume_backend_name', ['tripleo_ceph']), $backend_availability_zone = hiera('cinder::backend::rbd::backend_availability_zone', undef), # lint:ignore:parameter_documentation $cinder_rbd_backend_host = hiera('cinder::backend_host', hiera('cinder::host', $::hostname)), @@ -73,31 +77,52 @@ class tripleo::profile::base::cinder::volume::rbd ( $cinder_rbd_secret_uuid = undef, $cinder_rbd_user_name = 'openstack', $cinder_rbd_flatten_volume_from_snapshot = hiera('cinder::backend::rbd::flatten_volume_from_snapshot', undef), + $multi_config = {}, $step = Integer(hiera('step')), ) { include tripleo::profile::base::cinder::volume if $step >= 4 { - cinder::backend::rbd { $backend_name : - backend_availability_zone => $backend_availability_zone, - backend_host => $cinder_rbd_backend_host, - rbd_ceph_conf => $cinder_rbd_ceph_conf, - rbd_pool => $cinder_rbd_pool_name, - rbd_user => $cinder_rbd_user_name, - rbd_secret_uuid => $cinder_rbd_secret_uuid, - rbd_flatten_volume_from_snapshot => $cinder_rbd_flatten_volume_from_snapshot, + $backend_defaults = { + 'CephClusterFSID' => $cinder_rbd_secret_uuid, + 'CephClientUserName' => $cinder_rbd_user_name, + 'CinderRbdAvailabilityZone' => $backend_availability_zone, + 'CinderRbdPoolName' => $cinder_rbd_pool_name, + 'CinderRbdExtraPools' => $cinder_rbd_extra_pools, + 'CinderRbdFlattenVolumeFromSnapshot' => $cinder_rbd_flatten_volume_from_snapshot, } - if $cinder_rbd_extra_pools { - $cinder_rbd_extra_pools.each |$pool_name| { - cinder::backend::rbd { "${backend_name}_${pool_name}" : - backend_availability_zone => $backend_availability_zone, + any2array($backend_name).each |String $backend| { + $backend_multi_config = pick($multi_config[$backend], {}) + + $multi_config_cluster = $backend_multi_config['CephClusterName'] + if $multi_config_cluster { + $backend_ceph_conf = "/etc/ceph/${multi_config_cluster}.conf" + } else { + $backend_ceph_conf = $cinder_rbd_ceph_conf + } + + $backend_config = merge($backend_defaults, $backend_multi_config) + + cinder::backend::rbd { $backend : + backend_availability_zone => $backend_config['CinderRbdAvailabilityZone'], + backend_host => $cinder_rbd_backend_host, + rbd_ceph_conf => $backend_ceph_conf, + rbd_pool => $backend_config['CinderRbdPoolName'], + rbd_user => $backend_config['CephClientUserName'], + rbd_secret_uuid => $backend_config['CephClusterFSID'], + rbd_flatten_volume_from_snapshot => $backend_config['CinderRbdFlattenVolumeFromSnapshot'], + } + + any2array($backend_config['CinderRbdExtraPools']).each |String $pool_name| { + cinder::backend::rbd { "${backend}_${pool_name}" : + backend_availability_zone => $backend_config['CinderRbdAvailabilityZone'], backend_host => $cinder_rbd_backend_host, - rbd_ceph_conf => $cinder_rbd_ceph_conf, + rbd_ceph_conf => $backend_ceph_conf, rbd_pool => $pool_name, - rbd_user => $cinder_rbd_user_name, - rbd_secret_uuid => $cinder_rbd_secret_uuid, - rbd_flatten_volume_from_snapshot => $cinder_rbd_flatten_volume_from_snapshot, + rbd_user => $backend_config['CephClientUserName'], + rbd_secret_uuid => $backend_config['CephClusterFSID'], + rbd_flatten_volume_from_snapshot => $backend_config['CinderRbdFlattenVolumeFromSnapshot'], } } } diff --git a/releasenotes/notes/cinder-rbd-multiconfig-285d1542ef08fa10.yaml b/releasenotes/notes/cinder-rbd-multiconfig-285d1542ef08fa10.yaml new file mode 100644 index 000000000..41e2809d1 --- /dev/null +++ b/releasenotes/notes/cinder-rbd-multiconfig-285d1542ef08fa10.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add support for configuring multiple cinder RBD backends, each associated + with a different ceph cluster. + diff --git a/spec/classes/tripleo_profile_base_cinder_volume_rbd_spec.rb b/spec/classes/tripleo_profile_base_cinder_volume_rbd_spec.rb index 44e802687..ef7cfb8e3 100644 --- a/spec/classes/tripleo_profile_base_cinder_volume_rbd_spec.rb +++ b/spec/classes/tripleo_profile_base_cinder_volume_rbd_spec.rb @@ -94,6 +94,66 @@ describe 'tripleo::profile::base::cinder::volume::rbd' do ) end end + + context 'with multiple backends' do + before :each do + params.merge!({ + :backend_name => ['rbd1', 'rbd2'], + :backend_availability_zone => 'zone1', + :multi_config => { + 'rbd2' => { + 'CinderRbdAvailabilityZone' => 'zone2', + 'CephClusterName' => 'ceph2', + 'CinderRbdPoolName' => 'pool2a', + 'CinderRbdExtraPools' => ['pool2b', 'pool2c'], + 'CephClusterFSID' => 'secretuuid', + 'CephClientUserName' => 'kcatsnepo', + 'CinderRbdFlattenVolumeFromSnapshot' => true, + }, + }, + }) + end + it 'should configure each backend' do + is_expected.to contain_cinder__backend__rbd('rbd1').with( + :backend_host => 'node.example.com', + :backend_availability_zone => 'zone1', + :rbd_ceph_conf => '/etc/ceph/ceph.conf', + :rbd_pool => 'volumes', + :rbd_user => 'openstack', + :rbd_flatten_volume_from_snapshot => '', + ) + + is_expected.to contain_cinder__backend__rbd('rbd2').with( + :backend_host => 'node.example.com', + :backend_availability_zone => 'zone2', + :rbd_ceph_conf => '/etc/ceph/ceph2.conf', + :rbd_pool => 'pool2a', + :rbd_user => 'kcatsnepo', + :rbd_secret_uuid => 'secretuuid', + :rbd_flatten_volume_from_snapshot => true, + ) + + is_expected.to contain_cinder__backend__rbd('rbd2_pool2b').with( + :backend_host => 'node.example.com', + :backend_availability_zone => 'zone2', + :rbd_ceph_conf => '/etc/ceph/ceph2.conf', + :rbd_pool => 'pool2b', + :rbd_user => 'kcatsnepo', + :rbd_secret_uuid => 'secretuuid', + :rbd_flatten_volume_from_snapshot => true, + ) + + is_expected.to contain_cinder__backend__rbd('rbd2_pool2c').with( + :backend_host => 'node.example.com', + :backend_availability_zone => 'zone2', + :rbd_ceph_conf => '/etc/ceph/ceph2.conf', + :rbd_pool => 'pool2c', + :rbd_user => 'kcatsnepo', + :rbd_secret_uuid => 'secretuuid', + :rbd_flatten_volume_from_snapshot => true, + ) + end + end end end diff --git a/spec/classes/tripleo_profile_base_cinder_volume_spec.rb b/spec/classes/tripleo_profile_base_cinder_volume_spec.rb index 65b095388..8c88722c3 100644 --- a/spec/classes/tripleo_profile_base_cinder_volume_spec.rb +++ b/spec/classes/tripleo_profile_base_cinder_volume_spec.rb @@ -333,7 +333,6 @@ describe 'tripleo::profile::base::cinder::volume' do params.merge!({ :cinder_enable_rbd_backend => true, :cinder_enable_iscsi_backend => false, - :cinder_rbd_client_name => 'openstack' }) end it 'should configure only ceph' do @@ -345,18 +344,15 @@ describe 'tripleo::profile::base::cinder::volume' do is_expected.to contain_class('cinder::backends').with( :enabled_backends => ['tripleo_ceph'] ) - is_expected.to contain_exec('exec-setfacl-openstack-cinder') - is_expected.to contain_exec('exec-setfacl-openstack-cinder-mask') end - context 'additional rbd pools' do - # The list of additional rbd pools is not an input, but instead comes - # from hiera. Step 4's hiera data doesn't define additional RBD pools, - # so test the feature by defining extra pools in step 5 (see - # ../fixtures/hieradata/step5.yaml). + context 'additional rbd pools and an additional rbd backend' do + # Step 5's hiera specifies two rbd backends, each with additional rbd pools let(:params) { { :step => 5 } } it 'should configure additional rbd backends' do is_expected.to contain_class('cinder::backends').with( - :enabled_backends => ['tripleo_ceph', 'tripleo_ceph_foo', 'tripleo_ceph_bar'] + :enabled_backends => [ + 'tripleo_ceph_1', 'tripleo_ceph_2', 'tripleo_ceph_1_foo', 'tripleo_ceph_1_bar', 'tripleo_ceph_2_zap', + ] ) end end diff --git a/spec/fixtures/hieradata/step5.yaml b/spec/fixtures/hieradata/step5.yaml index 6da3d0a8d..222a48759 100644 --- a/spec/fixtures/hieradata/step5.yaml +++ b/spec/fixtures/hieradata/step5.yaml @@ -1,8 +1,15 @@ --- step: 5 +tripleo::profile::base::cinder::volume::rbd::backend_name: + - 'tripleo_ceph_1' + - 'tripleo_ceph_2' tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools: - 'foo' - 'bar' +tripleo::profile::base::cinder::volume::rbd::multi_config: + tripleo_ceph_2: + CinderRbdExtraPools: + - 'zap' cinder::backend::pure::volume_backend_name: - 'tripleo_pure_1' - 'tripleo_pure_2'