Define cephx libvirt secret for rbd volumes

This commit addresses the fact that any compute-worker needs to be

- prepared to talk to the ceph cluster (ceph.conf and ceph-common)
- a secret has to be defined.

The secret is identified by the UUID (known to cinder), as defined by a
databag item (of the "secrets" data bag) with a configurable name.

The actual key is taken from another databag item.  The username is
configured via the `rbd_user`.

The chefspec test on the proper content of the generated temporary xml
file is commented out since it won't work this way: the chefspec chef
run will have deleted the (in-memory representation of the) file when it
validates the spec, hence it's empty and the test fails.  (Not sure how
to fix that test.)

Implements: blueprint rbd-for-block-storage

Change-Id: I9eecc622b4d00c65fecfde0626f574be2b9ee934

README.md

@@ -54,6 +54,11 @@ libvirt
 ----
 - Installs libvirt, used by nova compute for management of the virtual machine environment
 
+libvirt_rbd
+----
+- Prepares the compute node for interaction with a Ceph cluster for block storage (RBD)
+- Depends on `openstack-common::ceph_client` for packages and cluster connectivity (i.e. a proper `/etc/ceph/ceph.conf`)
+
 network
 ----
 - Includes recipe `nova-common`
@@ -213,6 +218,7 @@ Libvirt Configuration Attributes
 ---------------------------------
 
 * `openstack["compute"]["libvirt"]["virt_type"]` - What hypervisor software layer to use with libvirt (e.g., kvm, qemu)
+* `openstack["compute"]["libvirt"]["volume_backend"]` - What block storage backend to use with libvirt (e.g. rbd)
 * `openstack["compute"]["libvirt"]["bind_interface"]` - Determine the interface's IP address (used for VNC).  IP address on the hypervisor that libvirt listens for VNC requests on, and IP address on the hypervisor that libvirt exposes for VNC requests on.
 * `openstack["compute"]["libvirt"]["auth_tcp"]` - Type of authentication your libvirt layer requires
 * `openstack["compute"]["libvirt"]["ssh"]["private_key"]` - Private key to use if using SSH authentication to your libvirt layer
@@ -227,6 +233,8 @@ Libvirt Configuration Attributes
 * `openstack["compute"]["libvirt"]["sparse_logical_volumes"]` - When images_type is lvm: use sparse logical volumes
 * `openstack["compute"]["libvirt"]["images_rbd_pool"]` - When images_type is rbd: use this RBD pool
 * `openstack["compute"]["libvirt"]["images_rbd_ceph_conf"]` - When images_type is rbd: use this ceph.conf
+* `openstack["compute"]["libvirt"]["rbd"]["rbd_user"]` - The cephx user used for accessing the RBD pool used for block storage. (Which pool to use is passed by cinder when nova-compute is instructed to mount a volume.)
+* `openstack["compute"]["libvirt"]["rbd"]["rbd_secret_name"]` - The name of the databag item containing the UUID shared between Cinder and nova-compute.  `libvirt_rbd` will define a libvirt secret with this UUID, containing the `rbd_user`'s password.  The password itself will be retrieved using `get_password` on the service `rbd_block_storage`.  Creating the cephx user in a Ceph cluster has to be done outside of the scope of this cookbook.
 
 Scheduler Configuration Attributes
 ----------------------------------

attributes/default.rb

@@ -192,7 +192,10 @@ default['openstack']['compute']['libvirt']['sparse_logical_volumes'] = false
 # rbd
 default['openstack']['compute']['libvirt']['images_rbd_pool'] = 'rbd'
 default['openstack']['compute']['libvirt']['images_rbd_ceph_conf'] = '/etc/ceph/ceph.conf'
-
+# use a different backend for volumes, allowed options: rbd
+default['openstack']['compute']['libvirt']['volume_backend'] = nil
+default['openstack']['compute']['libvirt']['rbd']['rbd_secret_name'] = 'rbd_secret_uuid'
+default['openstack']['compute']['libvirt']['rbd']['rbd_user'] = 'cinder'
 default['openstack']['compute']['config']['availability_zone'] = 'nova'
 default['openstack']['compute']['config']['storage_availability_zone'] = 'nova'
 default['openstack']['compute']['config']['default_schedule_zone'] = 'nova'
@@ -296,6 +299,7 @@ when 'fedora', 'redhat', 'centos', 'suse' # :pragma-foodcritic: ~FC024 - won't f
     'compute_vncproxy_consoleauth_service' => 'openstack-nova-consoleauth',
     'libvirt_packages' => ['libvirt'],
     'libvirt_service' => 'libvirtd',
+    'libvirt_ceph_packages' => ['ceph-common'],
     'dbus_service' => 'messagebus',
     'compute_cert_packages' => ['openstack-nova-cert'],
     'compute_cert_service' => 'openstack-nova-cert',
@@ -348,6 +352,7 @@ when 'ubuntu'
     'compute_vncproxy_consoleauth_service' => 'nova-consoleauth',
     'libvirt_packages' => ['libvirt-bin'],
     'libvirt_service' => 'libvirt-bin',
+    'libvirt_ceph_packages' => ['ceph-common'],
     'dbus_service' => 'dbus',
     'compute_cert_packages' => ['nova-cert'],
     'compute_cert_service' => 'nova-cert',

recipes/libvirt.rb

@@ -209,3 +209,6 @@ template '/etc/sysconfig/libvirtd' do
 
   only_if { platform? %w{fedora redhat centos} }
 end
+
+volume_backend = node['openstack']['compute']['libvirt']['volume_backend']
+include_recipe "openstack-compute::libvirt_#{volume_backend}" unless volume_backend.nil?

recipes/libvirt_rbd.rb

@@ -0,0 +1,67 @@
+# encoding: UTF-8
+#
+# Cookbook Name:: openstack-compute
+# Recipe:: libvirt_rbd
+#
+# Copyright 2014, x-ion GmbH
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class ::Chef::Recipe # rubocop:disable Documentation
+  include ::Openstack
+end
+
+# include_recipe 'openstack-common::ceph_client'
+
+platform_options = node['openstack']['compute']['platform']
+
+platform_options['libvirt_ceph_packages'].each do |pkg|
+  package pkg do
+    action :install
+  end
+end
+
+# TODO(srenatus) there might be multiple secrets, cinder will tell nova-compute
+# which one should be used for each single volume mount request
+Chef::Log.info("rbd_secret_name: #{node['openstack']['compute']['libvirt']['rbd']['rbd_secret_name']}")
+secret_uuid = secret 'secrets', node['openstack']['compute']['libvirt']['rbd']['rbd_secret_name']
+ceph_key = get_password 'service', 'rbd_block_storage'
+
+require 'securerandom'
+filename = SecureRandom.hex
+
+template "/tmp/#{filename}.xml" do
+  source 'secret.xml.erb'
+  user 'root'
+  group 'root'
+  mode '700'
+  variables(
+    uuid: secret_uuid,
+    client_name: node['openstack']['compute']['libvirt']['rbd']['rbd_user']
+  )
+  not_if "virsh secret-list | grep #{secret_uuid}"
+end
+
+execute "virsh secret-define --file /tmp/#{filename}.xml" do
+  not_if "virsh secret-list | grep #{secret_uuid}"
+end
+
+# this will update the key if necessary
+execute "virsh secret-set-value --secret #{secret_uuid} '#{ceph_key}'" do
+  not_if "virsh secret-get-value #{secret_uuid} | grep '#{ceph_key}'"
+end
+
+file "/tmp/#{filename}.xml" do
+  action :delete
+end

spec/libvirt_rbd_spec.rb

@@ -0,0 +1,65 @@
+# encoding: UTF-8
+
+require_relative 'spec_helper'
+
+describe 'openstack-compute::libvirt_rbd' do
+  before { compute_stubs }
+  describe 'ubuntu' do
+    before do
+      @chef_run = ::ChefSpec::Runner.new(::UBUNTU_OPTS) do |n|
+        n.set['openstack']['compute']['libvirt']['volume_backend'] = 'rbd'
+      end
+      @chef_run.converge 'openstack-compute::libvirt_rbd'
+    end
+
+    it 'includes the openstack-common::ceph_client recipe' do
+      pending 'TODO: openstack-common needs that recipe first'
+      expect(@chef_run).to include_recipe('openstack-common::ceph_client')
+    end
+
+    it 'installs rbd packages' do
+      expect(@chef_run).to install_package 'ceph-common'
+    end
+
+    describe 'if there was no secret with this uuid defined' do
+      before do
+        @filename = '/tmp/ad3313264ea51d8c6a3d1c5b140b9883.xml'
+      end
+
+      it 'creates the temporary secret xml file' do
+        expect(@chef_run).to create_template(@filename).with(
+          owner: 'root',
+          group: 'root',
+          mode: '700'
+        )
+        # TODO(srenatus) cannot check for its contents because it's deleted at
+        # the end of the (chefspec) chef run.
+        # [/client\.cinder secret/,
+        #  /00000000-0000-0000-0000-000000000000/].each do |content|
+        #   expect(@chef_run).to render_file(@filename).with_content(content)
+        # end
+      end
+
+      it 'defines the secret' do
+        expect(@chef_run).to run_execute('virsh secret-define --file /tmp/ad3313264ea51d8c6a3d1c5b140b9883.xml')
+      end
+
+      it 'sets the secret value to the password' do
+        expect(@chef_run).to run_execute("virsh secret-set-value --secret 00000000-0000-0000-0000-000000000000 'cinder-rbd-pass'")
+      end
+
+      it 'deletes the temporary secret xml file' do
+        expect(@chef_run).to delete_file(@filename)
+      end
+
+    end
+
+    # TODO(srenatus) negative tests?
+    # describe 'if the secret was already defined' do
+    #   before do
+    #     stub_command('virsh secret-list | grep 00000000-0000-0000-0000-000000000000').and_return(true)
+    #     stub_command('virsh secret-get-value 00000000-0000-0000-0000-000000000000 | grep \'cinder-rbd-pass\'').and_return(true)
+    #   end
+    # end
+  end
+end

spec/libvirt_spec.rb

@@ -46,6 +46,19 @@ describe 'openstack-compute::libvirt' do
       expect(@chef_run).to run_execute('virsh net-destroy default')
     end
 
+    describe 'rbd/ceph volume storage' do
+      before do
+        @chef_run = ::ChefSpec::Runner.new(::UBUNTU_OPTS) do |n|
+          n.set['openstack']['compute']['libvirt']['volume_backend'] = 'rbd'
+        end
+        @chef_run.converge 'openstack-compute::libvirt'
+      end
+
+      it 'includes the libvirt_rbd recipe if it is the selected volume backend' do
+        expect(@chef_run).to include_recipe('openstack-compute::libvirt_rbd')
+      end
+    end
+
     describe '/etc/libvirt/libvirtd.conf' do
       before { @filename = '/etc/libvirt/libvirtd.conf' }
 

spec/spec_helper.rb

@@ -3,6 +3,7 @@
 require 'chefspec'
 require 'chefspec/berkshelf'
 require 'chef/application'
+require 'securerandom'
 
 ::LOG_LEVEL = :fatal
 ::SUSE_OPTS = {
@@ -44,6 +45,9 @@ def compute_stubs # rubocop:disable MethodLength
   ::Chef::Recipe.any_instance.stub(:secret)
     .with('secrets', 'neutron_metadata_secret')
     .and_return('metadata-secret')
+  ::Chef::Recipe.any_instance.stub(:secret) # this is the rbd_uuid default name
+    .with('secrets', 'rbd_secret_uuid')
+    .and_return '00000000-0000-0000-0000-000000000000'
   ::Chef::Recipe.any_instance.stub(:get_password)
     .with('db', anything)
     .and_return('')
@@ -59,17 +63,23 @@ def compute_stubs # rubocop:disable MethodLength
   ::Chef::Recipe.any_instance.stub(:get_password)
     .with('service', 'openstack-network')
     .and_return('neutron-pass')
+  ::Chef::Recipe.any_instance.stub(:get_password)
+    .with('service', 'rbd_block_storage')
+    .and_return 'cinder-rbd-pass'
   ::Chef::Recipe.any_instance.stub(:memcached_servers).and_return []
   ::Chef::Recipe.any_instance.stub(:system)
     .with("grub2-set-default 'openSUSE GNU/Linux, with Xen hypervisor'")
     .and_return(true)
   ::Chef::Application.stub(:fatal!)
+  ::SecureRandom.stub(:hex) { 'ad3313264ea51d8c6a3d1c5b140b9883' }
   stub_command('nova-manage network list | grep 192.168.100.0/24').and_return(false)
   stub_command('nova-manage network list | grep 192.168.200.0/24').and_return(false)
   stub_command("nova-manage floating list |grep -E '.*([0-9]{1,3}[.]){3}[0-9]{1,3}*'").and_return(false)
   stub_command('virsh net-list | grep -q default').and_return(true)
   stub_command("ovs-vsctl show | grep 'Bridge br-int'").and_return(true)
   stub_command("ovs-vsctl show | grep 'Bridge br-tun'").and_return(true)
+  stub_command('virsh secret-list | grep 00000000-0000-0000-0000-000000000000').and_return(false)
+  stub_command("virsh secret-get-value 00000000-0000-0000-0000-000000000000 | grep 'cinder-rbd-pass'").and_return(false)
 end
 
 def expect_runs_nova_common_recipe

templates/default/secret.xml.erb

@@ -0,0 +1,6 @@
+<secret ephemeral='no' private='no'>
+  <uuid><%= @uuid -%></uuid>
+  <usage type='ceph'>
+    <name>client.<%= @client_name -%> secret</name>
+  </usage>
+</secret>