Add dependency on upstream ceph cookbook for better key management

The upstream ceph cookbook already hooks into chef for environment information.
This patch utilizes the client LWRP to create or add ceph keys for RBD support.
This patch also changes some default attribute names for more sane organization

Partial-Bug: #1409943

Change-Id: Ia87e123ae2d88e3122b8d66d8ff144a804e6d8c4

Berksfile

@@ -10,3 +10,5 @@ cookbook "openstack-common",
   github: "stackforge/cookbook-openstack-common"
 cookbook "openstack-network",
   github: "stackforge/cookbook-openstack-network"
+cookbook "ceph",
+  github: "ceph/ceph-cookbook"

CHANGELOG.md

@@ -25,6 +25,7 @@ This file is used to list changes made in each version of cookbook-openstack-com
 * Allow dbsync_timeout to be configurable
 * Make libvirtd service started prior to nova compute service
 * Add [upgrade_levels] in nova.conf to enable rpc API compatible with order version
+* Add dependency on upstream ceph cookbook for better key management
 
 ## 9.3.1
 * Move auth configuration from api-paste.ini to nova.conf

README.md

@@ -56,7 +56,7 @@ libvirt
 libvirt_rbd
 ----
 - Prepares the compute node for interaction with a Ceph cluster for block storage (RBD)
-- Depends on `openstack-common::ceph_client` for packages and cluster connectivity (i.e. a proper `/etc/ceph/ceph.conf`)
+- Depends on `ceph::_common`, `ceph::install`, and `ceph::conf` for packages and cluster connectivity (i.e. a proper `/etc/ceph/ceph.conf`)
 
 network
 ----
@@ -264,15 +264,17 @@ Libvirt Configuration Attributes
 * `openstack["compute"]["libvirt"]["images_type"]` - How to store local images (ephemeral disks): raw, qcow2, lvm, rbd, or default
 * `openstack["compute"]["libvirt"]["volume_group"]` - When images_type is lvm: volume group to use
 * `openstack["compute"]["libvirt"]["sparse_logical_volumes"]` - When images_type is lvm: use sparse logical volumes
-* `openstack["compute"]["libvirt"]["images_rbd_pool"]` - When images_type is rbd: use this RBD pool
-* `openstack["compute"]["libvirt"]["images_rbd_ceph_conf"]` - When images_type is rbd: use this ceph.conf
 * `openstack["compute"]["libvirt"]["unix_sock_rw_perms"]` - Set the UNIX socket permissions for the R/W socket. This is used for full management of VMs.
 * `openstack["compute"]["libvirt"]["live_migration_bandwidth"]` - Maximum bandwidth to be used during migration, in Mbps.
 * `openstack["compute"]["libvirt"]["live_migration_flag"]` - Migration flags to be set for live migration.
 * `openstack["compute"]["libvirt"]["block_migration_flag"]` - Migration flags to be set for block migration.
 * `openstack["compute"]["libvirt"]["live_migration_uri"]` - Migration target URI (any included "%s" is replaced with the migration target hostname).
-* `openstack["compute"]["libvirt"]["rbd"]["rbd_user"]` - The cephx user used for accessing the RBD pool used for block storage. (Which pool to use is passed by cinder when nova-compute is instructed to mount a volume.)
-* `openstack["compute"]["libvirt"]["rbd"]["rbd_secret_name"]` - The name of the databag item containing the UUID shared between Cinder and nova-compute.  `libvirt_rbd` will define a libvirt secret with this UUID, containing the `rbd_user`'s password.  The password itself will be retrieved using `get_password` on the service `rbd_block_storage`.  Creating the cephx user in a Ceph cluster has to be done outside of the scope of this cookbook.
+* `openstack["compute"]["libvirt"]["rbd"]["glance"]["pool"]` - When images_type is rbd: use this RBD pool for images
+* `openstack["compute"]["libvirt"]["rbd"]["cinder"]["pool"]` - When images_type is rbd: use this RBD pool for volumes
+* `openstack["compute"]["libvirt"]["rbd"]["nova"]["pool"]` - When images_type is rbd: use this RBD pool for instances
+* `openstack["compute"]["libvirt"]["rbd"]["ceph_conf"]` - When images_type is rbd: use this ceph.conf
+* `openstack["compute"]["libvirt"]["rbd"]["cinder"]["user"]` - The cephx user used for accessing the RBD pool used for block storage. (Which pool to use is passed by cinder when nova-compute is instructed to mount a volume.)
+* `openstack["compute"]["libvirt"]["rbd"]["cinder"]["secret_uuid"]` - A shared secret between cinder and libvirt.  It should be the same as the secret_uuid that is defined in block-storage.
 * `openstack["compute"]["libvirt"]["rng_dev_path"]` - A path to a device that will be used as source of entropy on the host. Permitted options are: /dev/random or /dev/hwrng (string value)
 
 Scheduler Configuration Attributes

attributes/default.rb

@@ -268,15 +268,18 @@ default['openstack']['compute']['libvirt']['images_type'] = 'default'
 default['openstack']['compute']['libvirt']['volume_group'] = nil
 default['openstack']['compute']['libvirt']['sparse_logical_volumes'] = false
 # rbd
-default['openstack']['compute']['libvirt']['images_rbd_pool'] = 'rbd'
-default['openstack']['compute']['libvirt']['images_rbd_ceph_conf'] = '/etc/ceph/ceph.conf'
+default['openstack']['compute']['libvirt']['rbd']['ceph_conf'] = '/etc/ceph/ceph.conf'
 # use a different backend for volumes, allowed options: rbd
 default['openstack']['compute']['libvirt']['volume_backend'] = nil
-default['openstack']['compute']['libvirt']['rbd']['rbd_secret_name'] = 'rbd_secret_uuid'
-default['openstack']['compute']['libvirt']['rbd']['rbd_user'] = 'cinder'
+default['openstack']['compute']['libvirt']['rbd']['cinder']['pool'] = 'volumes'
+default['openstack']['compute']['libvirt']['rbd']['glance']['pool'] = 'images'
+default['openstack']['compute']['libvirt']['rbd']['nova']['pool'] = 'instances'
+
+default['openstack']['compute']['libvirt']['rbd']['cinder']['user'] = 'cinder'
+default['openstack']['compute']['libvirt']['rbd']['cinder']['secret_uuid'] = '00000000-0000-0000-0000-000000000000'
 # live migration
 default['openstack']['compute']['libvirt']['live_migration_bandwidth'] = 0
-default['openstack']['compute']['libvirt']['live_migration_flag'] = 'VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER'
+default['openstack']['compute']['libvirt']['live_migration_flag'] = 'VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST'
 default['openstack']['compute']['libvirt']['block_migration_flag'] = 'VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER, VIR_MIGRATE_NON_SHARED_INC'
 default['openstack']['compute']['libvirt']['live_migration_uri'] = 'qemu+tcp://%s/system'
 
@@ -343,7 +346,7 @@ default['openstack']['compute']['config']['quota_metadata_items'] = 128
 # megabytes of instance ram allowed per project (default: 51200)
 default['openstack']['compute']['config']['quota_ram'] = 51200
 # disk cache modes
-default['openstack']['compute']['config']['disk_cache_modes'] = nil
+default['openstack']['compute']['config']['disk_cachemodes'] = nil
 # Number of 1 second retries needed in live_migration
 default['openstack']['compute']['config']['live_migration_retry_count'] = 30
 

metadata.rb

@@ -13,6 +13,7 @@ recipe 'openstack-compute::client', 'Install nova client packages'
 recipe 'openstack-compute::compute', 'nova-compute service'
 recipe 'openstack-compute::conductor', 'Installs nova conductor service'
 recipe 'openstack-compute::libvirt', 'Installs libvirt, used by nova compute for management of the virtual machine environment'
+recipe 'openstack-compute::libvirt_rbd', 'Manages the RBD portions of libvirt. Included by openstack-compute::libvirt'
 recipe 'openstack-compute::identity_registration', 'Registers the API and EC2 endpoints with Keystone'
 recipe 'openstack-compute::network', 'Installs nova network service'
 recipe 'openstack-compute::nova-cert', 'Installs nova-cert service'
@@ -25,6 +26,8 @@ recipe 'openstack-compute::vncproxy', 'Installs and configures the vncproxy serv
   supports os
 end
 
+depends 'ceph', '>= 0.2.1'
+depends 'ceph', '< 3.0.0'
 depends 'openstack-common', '~> 10.2'
 depends 'openstack-identity', '~> 10.0'
 depends 'openstack-image', '~> 10.0'

recipes/libvirt_rbd.rb

@@ -22,47 +22,54 @@ class ::Chef::Recipe # rubocop:disable Documentation
   include ::Openstack
 end
 
-include_recipe 'openstack-common::ceph_client'
+include_recipe 'ceph'
 
-platform_options = node['openstack']['compute']['platform']
+ceph_user = node['openstack']['compute']['libvirt']['rbd']['cinder']['user']
+cinder_pool = node['openstack']['compute']['libvirt']['rbd']['cinder']['pool']
+nova_pool = node['openstack']['compute']['libvirt']['rbd']['nova']['pool']
+glance_pool =  node['openstack']['compute']['libvirt']['rbd']['glance']['pool']
 
-platform_options['libvirt_ceph_packages'].each do |pkg|
-  package pkg do
-    options platform_options['package_overrides']
-    action :upgrade
-  end
+secret_uuid = node['openstack']['compute']['libvirt']['rbd']['cinder']['secret_uuid']
+ceph_keyname = "client.#{ceph_user}"
+ceph_keyring = "/etc/ceph/ceph.#{ceph_keyname}.keyring"
+
+caps = { 'mon' => 'allow r',
+         'osd' => "allow class-read object_prefix rbd_children, allow rwx pool=#{cinder_pool}, allow rwx pool=#{nova_pool}, allow rx pool=#{glance_pool}" }
+
+ceph_client ceph_user do
+  name ceph_user
+  caps caps
+  keyname ceph_keyname
+  filename ceph_keyring
+  owner node['openstack']['compute']['user']
+  group node['openstack']['compute']['group']
+
+  action :add
 end
 
-# TODO(srenatus) there might be multiple secrets, cinder will tell nova-compute
-# which one should be used for each single volume mount request
-Chef::Log.info("rbd_secret_name: #{node['openstack']['compute']['libvirt']['rbd']['rbd_secret_name']}")
-secret_uuid = get_secret node['openstack']['compute']['libvirt']['rbd']['rbd_secret_name']
-ceph_key = get_password 'service', 'rbd_block_storage'
+Chef::Log.info("rbd_secret_name: #{secret_uuid}")
 
-require 'securerandom'
-filename = SecureRandom.hex
-
-template "/tmp/#{filename}.xml" do
+template '/tmp/secret.xml' do
   source 'secret.xml.erb'
   user 'root'
   group 'root'
-  mode '700'
+  mode '00600'
   variables(
     uuid: secret_uuid,
-    client_name: node['openstack']['compute']['libvirt']['rbd']['rbd_user']
+    client_name: node['openstack']['compute']['libvirt']['rbd']['cinder']['user']
   )
   not_if "virsh secret-list | grep #{secret_uuid}"
 end
 
-execute "virsh secret-define --file /tmp/#{filename}.xml" do
+execute 'virsh secret-define --file /tmp/secret.xml' do
   not_if "virsh secret-list | grep #{secret_uuid}"
 end
 
 # this will update the key if necessary
-execute "virsh secret-set-value --secret #{secret_uuid} '#{ceph_key}'" do
-  not_if "virsh secret-get-value #{secret_uuid} | grep '#{ceph_key}'"
+execute "virsh secret-set-value --secret #{secret_uuid} --base64 $(ceph-authtool -p -n client.#{ceph_user} #{ceph_keyring})" do
+  not_if "virsh secret-get-value #{secret_uuid} | grep $(ceph-authtool -p -n #{ceph_keyname} #{ceph_keyring})"
 end
 
-file "/tmp/#{filename}.xml" do
+file '/tmp/secret.xml' do
   action :delete
 end

recipes/nova-common.rb

@@ -117,10 +117,6 @@ if node['openstack']['compute']['network']['service_type'] == 'neutron'
   neutron_metadata_proxy_shared_secret = get_secret 'neutron_metadata_secret'
 end
 
-if node['openstack']['compute']['libvirt']['images_type'] == 'rbd'
-  rbd_secret_uuid = get_secret node['openstack']['compute']['libvirt']['rbd']['rbd_secret_name']
-end
-
 if node['openstack']['compute']['driver'].split('.').first == 'vmwareapi'
   vmware_host_pass = get_secret node['openstack']['compute']['vmware']['secret_name']
 end
@@ -164,7 +160,6 @@ template '/etc/nova/nova.conf' do
     compute_api_bind_port: compute_api_bind.port,
     ec2_api_bind_ip: ec2_api_bind.host,
     ec2_api_bind_port: ec2_api_bind.port,
-    rbd_secret_uuid: rbd_secret_uuid,
     vmware_host_pass: vmware_host_pass,
     auth_uri: auth_uri,
     identity_admin_endpoint: identity_admin_endpoint,

spec/libvirt_rbd_spec.rb

@@ -7,6 +7,7 @@ describe 'openstack-compute::libvirt_rbd' do
     let(:runner) { ChefSpec::Runner.new(UBUNTU_OPTS) }
     let(:node) { runner.node }
     let(:chef_run) do
+      node.set['ceph']['config']['fsid'] = '00000000-0000-0000-0000-000000000000'
       node.set['openstack']['compute']['libvirt']['volume_backend'] = 'rbd'
 
       runner.converge(described_recipe)
@@ -14,51 +15,32 @@ describe 'openstack-compute::libvirt_rbd' do
 
     include_context 'compute_stubs'
 
-    it 'includes the openstack-common::ceph_client recipe' do
-      expect(chef_run).to include_recipe('openstack-common::ceph_client')
-    end
-
-    it 'upgrades rbd packages' do
-      expect(chef_run).to upgrade_package 'ceph-common'
+    it 'includes the ceph recipe' do
+      expect(chef_run).to include_recipe('ceph')
     end
 
     describe 'if there was no secret with this uuid defined' do
-      let(:file) { chef_run.template('/tmp/ad3313264ea51d8c6a3d1c5b140b9883.xml') }
-
-      it 'creates the temporary secret xml file' do
-        expect(chef_run).to create_template(file.name).with(
-          owner: 'root',
-          group: 'root',
-          mode: '700'
-        )
-        # TODO(srenatus) cannot check for its contents because it's deleted at
-        # the end of the (chefspec) chef run.
-        # [/client\.cinder secret/,
-        #  /00000000-0000-0000-0000-000000000000/].each do |content|
-        #   expect(@chef_run).to render_file(@filename).with_content(content)
-        # end
-      end
+      let(:file) { chef_run.template('/tmp/secret.xml') }
 
       it 'defines the secret' do
-        expect(chef_run).to run_execute("virsh secret-define --file #{file.name}")
+        expect(chef_run).to run_execute('virsh secret-define --file /tmp/secret.xml')
       end
 
       it 'sets the secret value to the password' do
-        expect(chef_run).to run_execute("virsh secret-set-value --secret 00000000-0000-0000-0000-000000000000 'cinder-rbd-pass'")
+        expect(chef_run).to run_execute('virsh secret-set-value --secret 00000000-0000-0000-0000-000000000000 --base64 $(ceph-authtool -p -n client.cinder /etc/ceph/ceph.client.cinder.keyring)')
+      end
+
+      it 'creates the temporary secret xml file' do
+        expect(chef_run).to create_template('/tmp/secret.xml').with(
+          owner: 'root',
+          group: 'root',
+          mode: '00600'
+        )
       end
 
       it 'deletes the temporary secret xml file' do
-        expect(chef_run).to delete_file(file.name)
+        expect(chef_run).to delete_file('/tmp/secret.xml')
       end
-
     end
-
-    # TODO(srenatus) negative tests?
-    # describe 'if the secret was already defined' do
-    #   before do
-    #     stub_command('virsh secret-list | grep 00000000-0000-0000-0000-000000000000').and_return(true)
-    #     stub_command('virsh secret-get-value 00000000-0000-0000-0000-000000000000 | grep \'cinder-rbd-pass\'').and_return(true)
-    #   end
-    # end
   end
 end

spec/libvirt_spec.rb

@@ -52,6 +52,7 @@ describe 'openstack-compute::libvirt' do
       end
 
       it 'includes the libvirt_rbd recipe if it is the selected volume backend' do
+        node.set['ceph']['config']['fsid'] = '00000000-0000-0000-0000-000000000000'
         expect(chef_run).to include_recipe('openstack-compute::libvirt_rbd')
       end
     end

spec/nova-common_spec.rb

@@ -576,7 +576,7 @@ describe 'openstack-compute::nova-common' do
             /^inject_password=false$/,
             /^inject_partition=-2$/,
             /^live_migration_bandwidth=0$/,
-            /^live_migration_flag=VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER$/,
+            /^live_migration_flag=VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST$/,
             /^block_migration_flag=VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER, VIR_MIGRATE_NON_SHARED_INC$/,
             %r{live_migration_uri=qemu\+tcp://%s/system$}
           ].each do |line|
@@ -719,7 +719,7 @@ describe 'openstack-compute::nova-common' do
           it 'sets the libvirt * options correctly' do
             [
               /^images_type=rbd$/,
-              /^images_rbd_pool=rbd$/,
+              /^images_rbd_pool=images$/,
               %r{^images_rbd_ceph_conf=/etc/ceph/ceph.conf$},
               /^rbd_user=cinder$/,
               /^rbd_secret_uuid=00000000-0000-0000-0000-000000000000$/
@@ -733,8 +733,8 @@ describe 'openstack-compute::nova-common' do
         describe 'override rbd settings' do
           before do
             node.set['openstack']['compute']['libvirt']['images_type'] = 'rbd'
-            node.set['openstack']['compute']['libvirt']['images_rbd_pool'] = 'myrbd'
-            node.set['openstack']['compute']['libvirt']['images_rbd_ceph_conf'] = '/etc/myceph/ceph.conf'
+            node.set['openstack']['compute']['libvirt']['rbd']['glance']['pool'] = 'myrbd'
+            node.set['openstack']['compute']['libvirt']['rbd']['ceph_conf'] = '/etc/myceph/ceph.conf'
           end
 
           it 'sets the overridden libvirt options correctly' do

spec/spec_helper.rb

@@ -86,7 +86,8 @@ shared_context 'compute_stubs' do
     stub_command('ovs-vsctl br-exists br-int').and_return(true)
     stub_command('ovs-vsctl br-exists br-tun').and_return(true)
     stub_command('virsh secret-list | grep 00000000-0000-0000-0000-000000000000').and_return(false)
-    stub_command("virsh secret-get-value 00000000-0000-0000-0000-000000000000 | grep 'cinder-rbd-pass'").and_return(false)
+    stub_command('virsh secret-set-value --secret 00000000-0000-0000-0000-000000000000 --base64 $(ceph-authtool -p -n client.cinder /etc/ceph/ceph.client.cinder.keyring)').and_return(false)
+    stub_command('virsh secret-get-value 00000000-0000-0000-0000-000000000000 | grep $(ceph-authtool -p -n client.cinder /etc/ceph/ceph.client.cinder.keyring)').and_return(false)
   end
 end
 

templates/default/nova.conf.erb

@@ -589,8 +589,8 @@ images_volume_group=<%= node['openstack']['compute']['libvirt']['volume_group']
 sparse_logical_volumes=<%= node['openstack']['compute']['libvirt']['sparse_logical_volumes'] %>
 
 <% elsif node['openstack']['compute']['libvirt']['images_type'] == 'rbd' -%>
-images_rbd_pool=<%= node['openstack']['compute']['libvirt']['images_rbd_pool'] %>
-images_rbd_ceph_conf=<%= node['openstack']['compute']['libvirt']['images_rbd_ceph_conf'] %>
+images_rbd_pool=<%= node['openstack']['compute']['libvirt']['rbd']['glance']['pool'] %>
+images_rbd_ceph_conf=<%= node['openstack']['compute']['libvirt']['rbd']['ceph_conf'] %>
 
 #
 # Options defined in nova.virt.libvirt.volume
@@ -600,9 +600,9 @@ images_rbd_ceph_conf=<%= node['openstack']['compute']['libvirt']['images_rbd_cep
 # NOTE that if these two options are set here, it will override the rbd_user that cinder provides
 # for nova in netdisk_properties:
 # https://github.com/openstack/nova/blob/c15dff2e9978fe851c73e92ab7f9b46e27de81ba/nova/virt/libvirt/volume.py#L217-L229
-rbd_user=<%= node['openstack']['compute']['libvirt']['rbd']['rbd_user'] %>
+rbd_user=<%= node['openstack']['compute']['libvirt']['rbd']['cinder']['user'] %>
 # The libvirt UUID of the secret for the rbd images (string value)
-rbd_secret_uuid=<%= @rbd_secret_uuid %>
+rbd_secret_uuid=<%= node['openstack']['compute']['libvirt']['rbd']['cinder']['secret_uuid'] %>
 <% end -%>
 
 [keystone_authtoken]