Ceph RBD integration for openstack-block-storage

This allows using a Ceph cluster as cinder backend.  The necessary
dependencies are defined in platform_options['cinder_ceph_packages'] and
can be overridden.

Overall documentation on Ceph integration has to be added, I'd be glad
to help with that.

This changes the way secrets are handled:

openstack[block-storage][rbd_secret_name]  will fetch the secret uuid
shared between cinder and nova from the databag with that name and set the
openstack[block-storage][rbd_secret_uuid] attribute.

Besides that, the user is expected to provide the password for the cephx
user, `rbd_user`, used with the pool `rbd_pool`, in the data bag item
`rbd_key_name` in the service password data bag.

Implements: blueprint rbd-for-block-storage

Change-Id: Iafe2bdea0d6120bb08c027482c94d819a67aee4a

attributes/default.rb

@@ -147,8 +147,11 @@ default['openstack']['block-storage']['volume']['iscsi_port'] = '3260'
 
 # Ceph/RADOS options
 default['openstack']['block-storage']['rbd_pool'] = 'rbd'
-default['openstack']['block-storage']['rbd_user'] = nil
+default['openstack']['block-storage']['rbd_user'] = 'cinder'
 default['openstack']['block-storage']['rbd_secret_uuid'] = nil
+# make this a valid uuid for when node['openstack']['developer_mode'] = true
+default['openstack']['block-storage']['rbd_secret_name'] = '00000000-0000-0000-0000-000000000000'
+default['openstack']['block-storage']['rbd_key_name'] = 'openstack_image_cephx_key'
 
 # Cinder Policy defaults
 default['openstack']['block-storage']['policy']['context_is_admin'] = '["role:admin"]'
@@ -175,6 +178,7 @@ when 'fedora', 'redhat', 'centos' # :pragma-foodcritic: ~FC024 - won't fix this
     'cinder_scheduler_service' => 'openstack-cinder-scheduler',
     'cinder_iscsitarget_packages' => ['scsi-target-utils'],
     'cinder_iscsitarget_service' => 'tgtd',
+    'cinder_ceph_packages' => ['python-ceph'],
     'cinder_nfs_packages' => ['nfs-utils', 'nfs-utils-lib'],
     'package_overrides' => ''
   }
@@ -192,6 +196,7 @@ when 'suse'
     'cinder_scheduler_service' => 'openstack-cinder-scheduler',
     'cinder_volume_packages' => ['openstack-cinder-volume'],
     'cinder_volume_service' => 'openstack-cinder-volume',
+    'cinder_ceph_packages' => ['python-ceph'],
     'cinder_iscsitarget_packages' => ['tgt'],
     'cinder_iscsitarget_service' => 'tgtd',
     'cinder_nfs_packages' => ['nfs-utils']
@@ -210,6 +215,7 @@ when 'ubuntu'
     'cinder_volume_service' => 'cinder-volume',
     'cinder_scheduler_packages' => ['cinder-scheduler'],
     'cinder_scheduler_service' => 'cinder-scheduler',
+    'cinder_ceph_packages' => ['python-ceph'],
     'cinder_iscsitarget_packages' => ['tgt'],
     'cinder_iscsitarget_service' => 'tgt',
     'cinder_nfs_packages' => ['nfs-common'],

recipes/volume.rb

@@ -56,7 +56,32 @@ when 'cinder.volume.drivers.netapp.iscsi.NetAppISCSIDriver'
   node.override['openstack']['block-storage']['netapp']['dfm_password'] = get_password 'service', 'netapp'
 
 when 'cinder.volume.drivers.rbd.RBDDriver'
-  node.override['openstack']['block-storage']['rbd_secret_uuid'] = get_password 'service', 'rbd'
+  # this is used in the cinder.conf template
+  node.override['openstack']['block-storage']['rbd_secret_uuid'] = secret 'secrets', node['openstack']['block-storage']['rbd_secret_name']
+
+  rbd_user = node['openstack']['block-storage']['rbd_user']
+  rbd_key = get_password 'service', node['openstack']['block-storage']['rbd_key_name']
+
+  include_recipe 'openstack-common::ceph_client'
+
+  platform_options['cinder_ceph_packages'].each do |pkg|
+    package pkg do
+      options platform_options['package_overrides']
+      action :install
+    end
+  end
+
+  template "/etc/ceph/ceph.client.#{rbd_user}.keyring" do
+    source 'ceph.client.keyring.erb'
+    cookbook 'openstack-common'
+    owner node['openstack']['block-storage']['user']
+    group node['openstack']['block-storage']['group']
+    mode '0600'
+    variables(
+      name: rbd_user,
+      key: rbd_key
+    )
+  end
 
 when 'cinder.volume.drivers.netapp.nfs.NetAppDirect7modeNfsDriver'
   node.override['openstack']['block-storage']['netapp']['netapp_server_password'] = get_password 'service', 'netapp-filer'

spec/spec_helper.rb

@@ -36,12 +36,18 @@ def block_storage_stubs # rubocop:disable MethodLength
   ::Chef::Recipe.any_instance.stub(:secret)
     .with('secrets', 'openstack_identity_bootstrap_token')
     .and_return('bootstrap-token')
+  ::Chef::Recipe.any_instance.stub(:secret)
+    .with('secrets', 'rbd_secret_uuid')
+    .and_return('b0ff3bba-e07b-49b1-beed-09a45552b1ad')
   ::Chef::Recipe.any_instance.stub(:get_password)
     .with('user', 'guest')
     .and_return('rabbit-pass')
   ::Chef::Recipe.any_instance.stub(:get_password)
     .with('service', 'openstack-block-storage')
     .and_return('cinder-pass')
+  ::Chef::Recipe.any_instance.stub(:get_password)
+    .with('service', 'openstack_image_cephx_key')
+    .and_return('cephx-key')
   ::Chef::Application.stub(:fatal!)
 end
 

spec/volume_spec.rb

@@ -75,16 +75,68 @@ describe 'openstack-block-storage::volume' do
       expect(n).to eq 'netapp-pass'
     end
 
-    it 'configures rbd password' do
-      ::Chef::Recipe.any_instance.stub(:get_password).with('service', 'rbd')
-        .and_return 'rbd-pass'
-      chef_run = ::ChefSpec::Runner.new ::UBUNTU_OPTS do |n|
-        n.set['openstack']['block-storage']['volume']['driver'] = 'cinder.volume.drivers.rbd.RBDDriver'
+    describe 'RBD Ceph as block-storage backend' do
+      before do
+        @chef_run = ::ChefSpec::Runner.new ::UBUNTU_OPTS do |n|
+          n.set['openstack']['block-storage']['volume']['driver'] = 'cinder.volume.drivers.rbd.RBDDriver'
+          n.set['openstack']['block-storage']['rbd_secret_name'] = 'rbd_secret_uuid'
+          # TODO: Remove work around once https://github.com/customink/fauxhai/pull/77 merges
+          n.set['cpu']['total'] = 1
+        end
+        @chef_run.converge 'openstack-block-storage::volume'
       end
-      chef_run.converge 'openstack-block-storage::volume'
-      n = chef_run.node['openstack']['block-storage']['rbd_secret_uuid']
 
-      expect(n).to eq 'rbd-pass'
+      it 'fetches the rbd_uuid_secret' do
+        n = @chef_run.node['openstack']['block-storage']['rbd_secret_uuid']
+        expect(n).to eq 'b0ff3bba-e07b-49b1-beed-09a45552b1ad'
+      end
+
+      it 'includes the ceph_client recipe' do
+        expect(@chef_run).to include_recipe('openstack-common::ceph_client')
+      end
+
+      it 'installs the python-ceph package by default' do
+        expect(@chef_run).to install_package('python-ceph')
+      end
+
+      it 'honors package option platform overrides for python-ceph' do
+        @chef_run = ::ChefSpec::Runner.new ::UBUNTU_OPTS do |n|
+          n.set['openstack']['block-storage']['volume']['driver'] = 'cinder.volume.drivers.rbd.RBDDriver'
+          n.set['openstack']['block-storage']['rbd_secret_name'] = 'rbd_secret_uuid'
+          n.set['openstack']['block-storage']['platform']['package_overrides'] = '-o Dpkg::Options::=\'--force-confold\' -o Dpkg::Options::=\'--force-confdef\' --force-yes'
+        end
+        @chef_run.converge 'openstack-block-storage::volume'
+
+        expect(@chef_run).to install_package('python-ceph').with(options: '-o Dpkg::Options::=\'--force-confold\' -o Dpkg::Options::=\'--force-confdef\' --force-yes')
+      end
+
+      it 'honors package name platform overrides for python-ceph' do
+        @chef_run = ::ChefSpec::Runner.new ::UBUNTU_OPTS do |n|
+          n.set['openstack']['block-storage']['volume']['driver'] = 'cinder.volume.drivers.rbd.RBDDriver'
+          n.set['openstack']['block-storage']['rbd_secret_name'] = 'rbd_secret_uuid'
+          n.set['openstack']['block-storage']['platform']['cinder_ceph_packages'] = ['my-ceph', 'my-other-ceph']
+        end
+        @chef_run.converge 'openstack-block-storage::volume'
+
+        %w{my-ceph my-other-ceph}.each do |pkg|
+          expect(@chef_run).to install_package(pkg)
+        end
+      end
+
+      it 'creates a cephx client keyring' do
+        pending 'https://review.openstack.org/#/c/69368/'
+        @file = '/etc/ceph/ceph.client.cinder.keyring'
+        [/^\[client\.cinder\]$/,
+         /key = cephx-key$/].each do |content|
+          expect(@chef_run).to render_file(@file).with_content(content)
+          expect(@chef_run).to create_template(@file).with(
+            cookbook: 'openstack-common',
+            owner: 'cinder',
+            group: 'cinder',
+            mode: 0600
+          )
+        end
+      end
     end
 
     it 'configures storewize private key' do