Add support for Ceph as a Cinder and Nova backend

This commit aims to add support for Ceph as a cinder and a nova backend.

  * Allows creation of Ceph pools from heat (Default: volumes, vms)
  * Creates the proper ceph user and inject the keys
  * Applies the proper configuration in cinder.conf and nova.conf
  * Enable the backend out of the box

Co-Authored-By: Giulio Fidente <gfidente@redhat.com>

Change-Id: Ic17d7a665de81a8bab5e34035abe90eda4bc889f

compute.yaml

@@ -195,6 +195,10 @@ parameters:
   NovaComputeLibvirtType:
     type: string
     default: ''
+  NovaEnableRbdBackend:
+    default: false
+    description: Whether to enable or not the Rbd backend for Nova
+    type: boolean
   NovaPassword:
     default: unset
     description: The password for the nova service account, used by nova-api.

controller.yaml

@@ -28,6 +28,10 @@ parameters:
     default: true
     description: Whether to enable or not the Iscsi backend for Cinder
     type: boolean
+  CinderEnableRbdBackend:
+    default: false
+    description: Whether to enable or not the Rbd backend for Cinder
+    type: boolean
   CinderISCSIHelper:
     default: tgtadm
     description: The iSCSI helper to use with cinder.

overcloud-without-mergepy.yaml

@@ -218,6 +218,10 @@ parameters:
     description: The keystone auth secret.
     type: string
     hidden: true
+  CinderEnableRbdBackend:
+    default: false
+    description: Whether to enable or not the Rbd backend for Cinder
+    type: boolean
   CinderLVMLoopDeviceSize:
     default: 5000
     description: The size of the loopback file used by the cinder LVM driver.
@@ -447,6 +451,10 @@ parameters:
   NovaComputeLibvirtType:
     default: ''
     type: string
+  NovaEnableRbdBackend:
+    default: false
+    description: Whether to enable or not the Rbd backend for Nova
+    type: boolean
   NovaImage:
     type: string
     default: overcloud-compute
@@ -518,6 +526,7 @@ resources:
           CinderPassword: {get_param: CinderPassword}
           CinderISCSIHelper: {get_param: CinderISCSIHelper}
           CinderEnableIscsiBackend: {get_param: CinderEnableIscsiBackend}
+          CinderEnableRbdBackend: {get_param: CinderEnableRbdBackend}
           CloudName: {get_param: CloudName}
           ControlVirtualInterface: {get_param: ControlVirtualInterface}
           ControllerExtraConfig: {get_param: controllerExtraConfig}
@@ -622,6 +631,7 @@ resources:
           NovaComputeDriver: {get_param: NovaComputeDriver}
           NovaComputeExtraConfig: {get_param: NovaComputeExtraConfig}
           NovaComputeLibvirtType: {get_param: NovaComputeLibvirtType}
+          NovaEnableRbdBackend: {get_param: NovaEnableRbdBackend}
           NovaPublicIP: {get_attr: [PublicVirtualIP, fixed_ips, 0, ip_address]}
           NovaPassword: {get_param: NovaPassword}
           NtpServer: {get_param: NtpServer}
@@ -754,6 +764,12 @@ resources:
       object_store_swift_devices: {get_attr: [ObjectStorage, swift_device]}
       controller_swift_proxy_memcaches: {get_attr: [Controller, swift_proxy_memcache]}
 
+  ComputeCephDeployment:
+    type: OS::Heat::StructuredDeployments
+    properties:
+      config: {get_attr: [CephClusterConfig, config_id]}
+      servers: {get_attr: [Compute, attributes, nova_server_resource]}
+
   ControllerCephDeployment:
     type: OS::Heat::StructuredDeployments
     properties:

puppet/compute-puppet.yaml

@@ -195,6 +195,10 @@ parameters:
   NovaComputeLibvirtType:
     type: string
     default: ''
+  NovaEnableRbdBackend:
+    default: false
+    description: Whether to enable or not the Rbd backend for Nova
+    type: boolean
   NovaPassword:
     default: unset
     description: The password for the nova service account, used by nova-api.
@@ -269,10 +273,14 @@ resources:
           hierarchy:
             - heat_config_%{::deploy_config_name}
             - compute
+            - ceph_cluster # provided by CephClusterConfig
+            - ceph
             - common
           datafiles:
             common:
               raw_data: {get_file: hieradata/common.yaml}
+            ceph:
+              raw_data: {get_file: hieradata/ceph.yaml}
             compute:
               raw_data: {get_file: hieradata/compute.yaml}
               oac_data:
@@ -283,6 +291,7 @@ resources:
                 nova::compute::libvirt::libvirt_virt_type: {get_input: nova_compute_libvirt_type}
                 nova_api_host: {get_input: nova_api_host}
                 nova::compute::vncproxy_host: {get_input: nova_public_ip}
+                nova_enable_rbd_backend: {get_input: nova_enable_rbd_backend}
                 nova_password: {get_input: nova_password}
                 #ceilometer::debug: {get_input: debug}
                 ceilometer::metering_secret: {get_input: ceilometer_metering_secret}
@@ -339,6 +348,7 @@ resources:
         nova_public_ip: {get_param: NovaPublicIP}
         nova_api_host: {get_param: NovaApiHost}
         nova_password: {get_param: NovaPassword}
+        nova_enable_rbd_backend: {get_param: NovaEnableRbdBackend}
         ceilometer_metering_secret: {get_param: CeilometerMeteringSecret}
         ceilometer_password: {get_param: CeilometerPassword}
         ceilometer_compute_agent: {get_param: CeilometerComputeAgent}

puppet/controller-puppet.yaml

@@ -28,6 +28,10 @@ parameters:
     default: true
     description: Whether to enable or not the Iscsi backend for Cinder
     type: boolean
+  CinderEnableRbdBackend:
+    default: false
+    description: Whether to enable or not the Rbd backend for Cinder
+    type: boolean
   CinderISCSIHelper:
     default: tgtadm
     description: The iSCSI helper to use with cinder.
@@ -445,6 +449,7 @@ resources:
         admin_token: {get_param: AdminToken}
         neutron_public_interface_ip: {get_param: NeutronPublicInterfaceIP}
         debug: {get_param: Debug}
+        cinder_enable_rbd_backend: {get_param: CinderEnableRbdBackend}
         cinder_lvm_loop_device_size: {get_param: CinderLVMLoopDeviceSize}
         cinder_password: {get_param: CinderPassword}
         cinder_enable_iscsi_backend: {get_param: CinderEnableIscsiBackend}
@@ -610,6 +615,7 @@ resources:
                 # See: https://review.openstack.org/#/c/109225/
                 tripleo::ringbuilder::build_ring: True
                 # Cinder
+                cinder_enable_rbd_backend: {get_input: cinder_enable_rbd_backend}
                 cinder_lvm_loop_device_size: {get_input: cinder_lvm_loop_device_size}
                 cinder_iscsi_helper: {get_input: cinder_iscsi_helper}
                 cinder_iscsi_ip_address: {get_input: controller_host}

puppet/hieradata/ceph.yaml

@@ -5,3 +5,9 @@ ceph::profile::params::osd_pool_default_size: 3
 ceph::profile::params::osd_pool_default_min_size: 1
 ceph::profile::params::manage_repo: false
 ceph::profile::params::authentication_type: cephx
+
+ceph_openstack_default_cap_mon: 'allow r'
+ceph_openstack_default_cap_osd: 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms'
+ceph_pools:
+  - volumes
+  - vms

puppet/hieradata/compute.yaml

@@ -10,6 +10,11 @@ nova::compute::vnc_enabled: true
 nova::compute::libvirt::vncserver_listen: '0.0.0.0'
 nova::compute::libvirt::migration_support: true
 
+nova::compute::rbd::libvirt_rbd_user: 'openstack'
+nova::compute::rbd::rbd_keyring: 'client.openstack'
+nova::compute::rbd::libvirt_images_rbd_pool: 'vms'
+nova::compute::rbd::libvirt_rbd_secret_uuid: "%{hiera('ceph::profile::params::fsid')}"
+
 neutron::plugins::ml2::tunnel_id_ranges: ['1:1000']
 
 ceilometer::agent::auth::auth_tenant_name: 'service'

puppet/manifests/overcloud_compute.pp

@@ -45,6 +45,18 @@ nova_config {
   'DEFAULT/linuxnet_interface_driver': value => 'nova.network.linux_net.LinuxOVSInterfaceDriver';
 }
 
+$nova_enable_rbd_backend = hiera('nova_enable_rbd_backend', false)
+if $nova_enable_rbd_backend {
+  include ::ceph::profile::client
+  include ::nova::compute::rbd
+  ceph::key { 'client.openstack' :
+    secret  => hiera('ceph::profile::params::mon_key'),
+    cap_mon => hiera('ceph_openstack_default_cap_mon'),
+    cap_osd => hiera('ceph_openstack_default_cap_osd'),
+    user    => 'nova',
+  }
+}
+
 include ::nova::compute::libvirt
 
 class { 'nova::network::neutron':

puppet/manifests/overcloud_controller.pp

@@ -156,14 +156,26 @@ if hiera('step') >= 2 {
   # pre-install swift here so we can build rings
   include ::swift
 
-  # don't install Ceph if FSID is not provided
-  if hiera('ceph::profile::params::fsid', false) {
+  $cinder_enable_rbd_backend = hiera('cinder_enable_rbd_backend', false)
+  $enable_ceph = $cinder_enable_rbd_backend
+
+  if $enable_ceph {
     class { 'ceph::profile::params':
       mon_initial_members => downcase(hiera('ceph_mon_initial_members'))
     }
     include ::ceph::profile::mon
   }
 
+  if $cinder_enable_rbd_backend {
+    ceph::key { 'client.openstack' :
+      secret  => hiera('ceph::profile::params::mon_key'),
+      cap_mon => hiera('ceph_openstack_default_cap_mon'),
+      cap_osd => hiera('ceph_openstack_default_cap_osd'),
+      user    => 'cinder',
+      inject  => 'true',
+    }
+  }
+
 } #END STEP 2
 
 if hiera('step') >= 3 {
@@ -278,7 +290,34 @@ if hiera('step') >= 3 {
     }
   }
 
-  $cinder_enabled_backends = any2array($cinder_iscsi_backend)
+  if $enable_ceph {
+
+    Ceph_pool {
+      pg_num  => hiera('ceph::profile::params::osd_pool_default_pg_num'),
+      pgp_num => hiera('ceph::profile::params::osd_pool_default_pgp_num'),
+      size    => hiera('ceph::profile::params::osd_pool_default_size'),
+    }
+
+    $ceph_pools = hiera('ceph_pools')
+    ceph::pool { $ceph_pools : }
+  }
+
+  if $cinder_enable_rbd_backend {
+    $cinder_rbd_backend = 'tripleo_ceph'
+
+    cinder_config {
+      "${cinder_rbd_backend}/host": value => 'hostgroup';
+    }
+
+    cinder::backend::rbd { $cinder_rbd_backend :
+      rbd_pool        => 'volumes',
+      rbd_user        => 'openstack',
+      rbd_secret_uuid => hiera('ceph::profile::params::fsid'),
+      require         => Ceph::Pool['volumes'],
+    }
+  }
+
+  $cinder_enabled_backends = concat(any2array($cinder_iscsi_backend), $cinder_rbd_backend)
   class { '::cinder::backends' :
     enabled_backends => $cinder_enabled_backends,
   }