Add cinder RBD support for multiple ceph clusters

Following the pattern of other cinder backends, a new tripleo::profile::base::cinder::volume::rbd::multi_config parameter adds support for configuring cinder RBD backends associated with multiple ceph clusters. The multi_config parameter is a hash that specifies the settings required to access each additional cluster (FSID, pool and client names, etc.). This patch also deprecates legacy code that managed ACLs for the ceph client keyring. THT already ensures cinder can access the keyring using kolla_config permissions. Change-Id: I040e25341c9869ad289d7e7c98e831caef23fece
2021-04-01 11:50:27 -07:00
parent 9d56b927c0
commit 22a5dd19bd
6 changed files with 153 additions and 55 deletions
--- a/manifests/profile/base/cinder/volume.pp
+++ b/manifests/profile/base/cinder/volume.pp
@@ -102,14 +102,6 @@
 #   (Optional) List of additional backend stanzas to activate
 #   Defaults to hiera('cinder_user_enabled_backends')
 #
-# [*cinder_rbd_client_name*]
-#   (Optional) Name of RBD client
-#   Defaults to hiera('tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name')
-#
-# [*cinder_rbd_ceph_conf_path*]
-#   (Optional) The path where the Ceph Cluster config files are stored on the host
-#   Defaults to '/etc/ceph'
-#
 # [*cinder_volume_cluster*]
 #   (Optional) Name of the cluster when running in active-active mode
 #   Defaults to ''
@@ -139,6 +131,16 @@
 #   for more details.
 #   Defaults to hiera('step')
 #
+# DEPRECATED PARAMETERS
+#
+# [*cinder_rbd_client_name*]
+#   (Optional) Name of RBD client
+#   Defaults to undef
+#
+# [*cinder_rbd_ceph_conf_path*]
+#   (Optional) The path where the Ceph Cluster config files are stored on the host
+#   Defaults to undef
+#
 class tripleo::profile::base::cinder::volume (
  $cinder_enable_pure_backend                  = false,
  $cinder_enable_dellsc_backend                = false,
@@ -161,8 +163,6 @@ class tripleo::profile::base::cinder::volume (
  $cinder_enable_vrts_hs_backend               = false,
  $cinder_enable_nvmeof_backend                = false,
  $cinder_user_enabled_backends                = hiera('cinder_user_enabled_backends', undef),
-  $cinder_rbd_ceph_conf_path                   = '/etc/ceph',
-  $cinder_rbd_client_name                      = hiera('tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name','openstack'),
  $cinder_volume_cluster                       = '',
  $enable_internal_tls                         = hiera('enable_internal_tls', false),
  $etcd_certificate_specs                      = hiera('tripleo::profile::base::etcd::certificate_specs', {}),
@@ -170,6 +170,9 @@ class tripleo::profile::base::cinder::volume (
  $etcd_host                                   = hiera('etcd_vip', undef),
  $etcd_port                                   = hiera('tripleo::profile::base::etcd::client_port', '2379'),
  $step                                        = Integer(hiera('step')),
+  # DEPRECATED PARAMETERS
+  $cinder_rbd_ceph_conf_path                   = undef,
+  $cinder_rbd_client_name                      = undef,
 ) {
  include tripleo::profile::base::cinder

@@ -326,26 +329,27 @@ class tripleo::profile::base::cinder::volume (

    if $cinder_enable_rbd_backend {
      include tripleo::profile::base::cinder::volume::rbd
-      $cinder_rbd_backend_name = hiera('cinder::backend::rbd::volume_backend_name', 'tripleo_ceph')
+      $cinder_rbd_backend_name = hiera('tripleo::profile::base::cinder::volume::rbd::backend_name',
+                                        ['tripleo_ceph'])

-      exec{ "exec-setfacl-${cinder_rbd_client_name}-cinder":
-        path    => ['/bin', '/usr/bin'],
-        command => "setfacl -m u:cinder:r-- ${cinder_rbd_ceph_conf_path}/ceph.client.${cinder_rbd_client_name}.keyring",
-        unless  => "getfacl ${cinder_rbd_ceph_conf_path}/ceph.client.${cinder_rbd_client_name}.keyring | grep -q user:cinder:r--",
-      }
-      -> exec{ "exec-setfacl-${cinder_rbd_client_name}-cinder-mask":
-        path    => ['/bin', '/usr/bin'],
-        command => "setfacl -m m::r ${cinder_rbd_ceph_conf_path}/ceph.client.${cinder_rbd_client_name}.keyring",
-        unless  => "getfacl ${cinder_rbd_ceph_conf_path}/ceph.client.${cinder_rbd_client_name}.keyring | grep -q mask::r",
-      }
-
-      $cinder_rbd_extra_pools = hiera('tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools', undef)
-      if $cinder_rbd_extra_pools {
-          $base_name = $cinder_rbd_backend_name
-          $cinder_rbd_extra_backend_names = $cinder_rbd_extra_pools.map |$pool_name| { "${base_name}_${pool_name}" }
+      $extra_pools = hiera('tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools', undef)
+      if empty($extra_pools) {
+        $extra_backend_names = []
      } else {
-          $cinder_rbd_extra_backend_names = undef
+        # These $extra_pools are associated with the first backend
+        $base_name = any2array($cinder_rbd_backend_name)[0]
+        $extra_backend_names = any2array($extra_pools).map |$pool_name| { "${base_name}_${pool_name}" }
      }
+
+      # Each $multi_config backend can specify its own list of extra pools. The
+      # backend names are the $multi_config hash keys.
+      $multi_config = hiera('tripleo::profile::base::cinder::volume::rbd::multi_config', {})
+      $extra_multiconfig_backend_names = $multi_config.map |$base_name, $backend_config| {
+        $backend_extra_pools = $backend_config['CinderRbdExtraPools']
+        any2array($backend_extra_pools).map |$pool_name| { "${base_name}_${pool_name}" }
+      }
+
+      $cinder_rbd_extra_backend_names = flatten($extra_backend_names, $extra_multiconfig_backend_names)
    } else {
      $cinder_rbd_backend_name = undef
      $cinder_rbd_extra_backend_names = undef
--- a/manifests/profile/base/cinder/volume/rbd.pp
+++ b/manifests/profile/base/cinder/volume/rbd.pp
@@ -19,8 +19,8 @@
 # === Parameters
 #
 # [*backend_name*]
-#   (Optional) Name given to the Cinder backend stanza
-#   Defaults to 'tripleo_ceph'
+#   (Optional)  List of names given to the Cinder backend stanza.
+#   Defaults to hiera('cinder::backend::rbd::volume_backend_name', ['tripleo_ceph'])
 #
 # [*backend_availability_zone*]
 #   (Optional) Availability zone for this volume backend
@@ -56,13 +56,17 @@
 #   in order to remove a dependency on the snapshot.
 #   Defaults to hiera('cinder::backend::rbd::flatten_volume_from_snapshot, undef)
 #
+# [*multi_config*]
+#   (Optional) A config hash when multiple backends are used.
+#   Defaults to {}
+#
 # [*step*]
 #   (Optional) The current step in deployment. See tripleo-heat-templates
 #   for more details.
 #   Defaults to hiera('step')
 #
 class tripleo::profile::base::cinder::volume::rbd (
-  $backend_name                            = hiera('cinder::backend::rbd::volume_backend_name', 'tripleo_ceph'),
+  $backend_name                            = hiera('cinder::backend::rbd::volume_backend_name', ['tripleo_ceph']),
  $backend_availability_zone               = hiera('cinder::backend::rbd::backend_availability_zone', undef),
  # lint:ignore:parameter_documentation
  $cinder_rbd_backend_host                 = hiera('cinder::backend_host', hiera('cinder::host', $::hostname)),
@@ -73,31 +77,52 @@ class tripleo::profile::base::cinder::volume::rbd (
  $cinder_rbd_secret_uuid                  = undef,
  $cinder_rbd_user_name                    = 'openstack',
  $cinder_rbd_flatten_volume_from_snapshot = hiera('cinder::backend::rbd::flatten_volume_from_snapshot', undef),
+  $multi_config                            = {},
  $step                                    = Integer(hiera('step')),
 ) {
  include tripleo::profile::base::cinder::volume

  if $step >= 4 {
-    cinder::backend::rbd { $backend_name :
-      backend_availability_zone        => $backend_availability_zone,
-      backend_host                     => $cinder_rbd_backend_host,
-      rbd_ceph_conf                    => $cinder_rbd_ceph_conf,
-      rbd_pool                         => $cinder_rbd_pool_name,
-      rbd_user                         => $cinder_rbd_user_name,
-      rbd_secret_uuid                  => $cinder_rbd_secret_uuid,
-      rbd_flatten_volume_from_snapshot => $cinder_rbd_flatten_volume_from_snapshot,
+    $backend_defaults = {
+      'CephClusterFSID'                    => $cinder_rbd_secret_uuid,
+      'CephClientUserName'                 => $cinder_rbd_user_name,
+      'CinderRbdAvailabilityZone'          => $backend_availability_zone,
+      'CinderRbdPoolName'                  => $cinder_rbd_pool_name,
+      'CinderRbdExtraPools'                => $cinder_rbd_extra_pools,
+      'CinderRbdFlattenVolumeFromSnapshot' => $cinder_rbd_flatten_volume_from_snapshot,
    }

-    if $cinder_rbd_extra_pools {
-      $cinder_rbd_extra_pools.each |$pool_name| {
-        cinder::backend::rbd { "${backend_name}_${pool_name}" :
-          backend_availability_zone        => $backend_availability_zone,
+    any2array($backend_name).each |String $backend| {
+      $backend_multi_config  = pick($multi_config[$backend], {})
+
+      $multi_config_cluster = $backend_multi_config['CephClusterName']
+      if $multi_config_cluster {
+        $backend_ceph_conf = "/etc/ceph/${multi_config_cluster}.conf"
+      } else {
+        $backend_ceph_conf = $cinder_rbd_ceph_conf
+      }
+
+      $backend_config = merge($backend_defaults, $backend_multi_config)
+
+      cinder::backend::rbd { $backend :
+        backend_availability_zone        => $backend_config['CinderRbdAvailabilityZone'],
+        backend_host                     => $cinder_rbd_backend_host,
+        rbd_ceph_conf                    => $backend_ceph_conf,
+        rbd_pool                         => $backend_config['CinderRbdPoolName'],
+        rbd_user                         => $backend_config['CephClientUserName'],
+        rbd_secret_uuid                  => $backend_config['CephClusterFSID'],
+        rbd_flatten_volume_from_snapshot => $backend_config['CinderRbdFlattenVolumeFromSnapshot'],
+      }
+
+      any2array($backend_config['CinderRbdExtraPools']).each |String $pool_name| {
+        cinder::backend::rbd { "${backend}_${pool_name}" :
+          backend_availability_zone        => $backend_config['CinderRbdAvailabilityZone'],
          backend_host                     => $cinder_rbd_backend_host,
-          rbd_ceph_conf                    => $cinder_rbd_ceph_conf,
+          rbd_ceph_conf                    => $backend_ceph_conf,
          rbd_pool                         => $pool_name,
-          rbd_user                         => $cinder_rbd_user_name,
-          rbd_secret_uuid                  => $cinder_rbd_secret_uuid,
-          rbd_flatten_volume_from_snapshot => $cinder_rbd_flatten_volume_from_snapshot,
+          rbd_user                         => $backend_config['CephClientUserName'],
+          rbd_secret_uuid                  => $backend_config['CephClusterFSID'],
+          rbd_flatten_volume_from_snapshot => $backend_config['CinderRbdFlattenVolumeFromSnapshot'],
        }
      }
    }
--- a/releasenotes/notes/cinder-rbd-multiconfig-285d1542ef08fa10.yaml
+++ b/releasenotes/notes/cinder-rbd-multiconfig-285d1542ef08fa10.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Add support for configuring multiple cinder RBD backends, each associated
+    with a different ceph cluster.
+    
--- a/spec/classes/tripleo_profile_base_cinder_volume_rbd_spec.rb
+++ b/spec/classes/tripleo_profile_base_cinder_volume_rbd_spec.rb
@@ -94,6 +94,66 @@ describe 'tripleo::profile::base::cinder::volume::rbd' do
          )
        end
      end
+
+      context 'with multiple backends' do
+        before :each do
+          params.merge!({
+            :backend_name              => ['rbd1', 'rbd2'],
+            :backend_availability_zone => 'zone1',
+            :multi_config              => {
+              'rbd2' => {
+                'CinderRbdAvailabilityZone'          => 'zone2',
+                'CephClusterName'                    => 'ceph2',
+                'CinderRbdPoolName'                  => 'pool2a',
+                'CinderRbdExtraPools'                => ['pool2b', 'pool2c'],
+                'CephClusterFSID'                    => 'secretuuid',
+                'CephClientUserName'                 => 'kcatsnepo',
+                'CinderRbdFlattenVolumeFromSnapshot' => true,
+              },
+            },
+          })
+        end
+        it 'should configure each backend' do
+          is_expected.to contain_cinder__backend__rbd('rbd1').with(
+            :backend_host                     => 'node.example.com',
+            :backend_availability_zone        => 'zone1',
+            :rbd_ceph_conf                    => '/etc/ceph/ceph.conf',
+            :rbd_pool                         => 'volumes',
+            :rbd_user                         => 'openstack',
+            :rbd_flatten_volume_from_snapshot => '<SERVICE DEFAULT>',
+          )
+
+          is_expected.to contain_cinder__backend__rbd('rbd2').with(
+            :backend_host                     => 'node.example.com',
+            :backend_availability_zone        => 'zone2',
+            :rbd_ceph_conf                    => '/etc/ceph/ceph2.conf',
+            :rbd_pool                         => 'pool2a',
+            :rbd_user                         => 'kcatsnepo',
+            :rbd_secret_uuid                  => 'secretuuid',
+            :rbd_flatten_volume_from_snapshot => true,
+          )
+
+          is_expected.to contain_cinder__backend__rbd('rbd2_pool2b').with(
+            :backend_host                     => 'node.example.com',
+            :backend_availability_zone        => 'zone2',
+            :rbd_ceph_conf                    => '/etc/ceph/ceph2.conf',
+            :rbd_pool                         => 'pool2b',
+            :rbd_user                         => 'kcatsnepo',
+            :rbd_secret_uuid                  => 'secretuuid',
+            :rbd_flatten_volume_from_snapshot => true,
+          )
+
+          is_expected.to contain_cinder__backend__rbd('rbd2_pool2c').with(
+            :backend_host                     => 'node.example.com',
+            :backend_availability_zone        => 'zone2',
+            :rbd_ceph_conf                    => '/etc/ceph/ceph2.conf',
+            :rbd_pool                         => 'pool2c',
+            :rbd_user                         => 'kcatsnepo',
+            :rbd_secret_uuid                  => 'secretuuid',
+            :rbd_flatten_volume_from_snapshot => true,
+          )
+        end
+      end
    end
  end

--- a/spec/classes/tripleo_profile_base_cinder_volume_spec.rb
+++ b/spec/classes/tripleo_profile_base_cinder_volume_spec.rb
@@ -333,7 +333,6 @@ describe 'tripleo::profile::base::cinder::volume' do
          params.merge!({
            :cinder_enable_rbd_backend   => true,
            :cinder_enable_iscsi_backend => false,
-            :cinder_rbd_client_name      => 'openstack'
          })
        end
        it 'should configure only ceph' do
@@ -345,18 +344,15 @@ describe 'tripleo::profile::base::cinder::volume' do
          is_expected.to contain_class('cinder::backends').with(
            :enabled_backends => ['tripleo_ceph']
          )
-          is_expected.to contain_exec('exec-setfacl-openstack-cinder')
-          is_expected.to contain_exec('exec-setfacl-openstack-cinder-mask')
        end
-        context 'additional rbd pools' do
-          # The list of additional rbd pools is not an input, but instead comes
-          # from hiera. Step 4's hiera data doesn't define additional RBD pools,
-          # so test the feature by defining extra pools in step 5 (see
-          # ../fixtures/hieradata/step5.yaml).
+        context 'additional rbd pools and an additional rbd backend' do
+          # Step 5's hiera specifies two rbd backends, each with additional rbd pools
          let(:params) { { :step => 5 } }
          it 'should configure additional rbd backends' do
            is_expected.to contain_class('cinder::backends').with(
-              :enabled_backends => ['tripleo_ceph', 'tripleo_ceph_foo', 'tripleo_ceph_bar']
+              :enabled_backends => [
+                'tripleo_ceph_1', 'tripleo_ceph_2', 'tripleo_ceph_1_foo', 'tripleo_ceph_1_bar', 'tripleo_ceph_2_zap',
+              ]
            )
          end
        end
--- a/spec/fixtures/hieradata/step5.yaml
+++ b/spec/fixtures/hieradata/step5.yaml
@@ -1,8 +1,15 @@
 ---
 step: 5
+tripleo::profile::base::cinder::volume::rbd::backend_name:
+  - 'tripleo_ceph_1'
+  - 'tripleo_ceph_2'
 tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools:
  - 'foo'
  - 'bar'
+tripleo::profile::base::cinder::volume::rbd::multi_config:
+  tripleo_ceph_2:
+    CinderRbdExtraPools:
+      - 'zap'
 cinder::backend::pure::volume_backend_name:
  - 'tripleo_pure_1'
  - 'tripleo_pure_2'