From 0fdb733b3f01f2474d1829301b45f387b7dfa48f Mon Sep 17 00:00:00 2001 From: Dmitry Borodaenko Date: Fri, 11 Oct 2013 00:21:14 -0700 Subject: [PATCH] cephx key and ceph pool management fixes * consistently parametrize pools, Cephx users and ACLs * unscatter creation of Cinder and Glance pools into ceph::pool * use parameterized Cephx user when generating Nova secret * create Glance pool before Cinder ACL --- deployment/puppet/ceph/manifests/init.pp | 72 +++++++++++-------- deployment/puppet/ceph/manifests/mon.pp | 12 +--- .../puppet/ceph/manifests/nova_compute.pp | 13 ++-- deployment/puppet/ceph/manifests/pool.pp | 45 ++++++++++++ .../puppet/cinder/manifests/volume/ceph.pp | 43 +++-------- .../puppet/glance/manifests/backend/ceph.pp | 37 ++-------- 6 files changed, 112 insertions(+), 110 deletions(-) create mode 100644 deployment/puppet/ceph/manifests/pool.pp diff --git a/deployment/puppet/ceph/manifests/init.pp b/deployment/puppet/ceph/manifests/init.pp index 17aaf28be0..c7618fea0c 100644 --- a/deployment/puppet/ceph/manifests/init.pp +++ b/deployment/puppet/ceph/manifests/init.pp @@ -2,25 +2,23 @@ class ceph ( # General settings - $cluster_node_address = $::ipaddress, #This should be the cluster service address - $primary_mon = $::hostname, #This should be the first controller - $cinder_pool = 'volumes', - $glance_pool = 'images', - $osd_devices = split($::osd_devices_list, ' '), - $use_ssl = false, - $use_rgw = false, + $cluster_node_address = $::ipaddress, #This should be the cluster service address + $primary_mon = $::hostname, #This should be the first controller + $osd_devices = split($::osd_devices_list, ' '), + $use_ssl = false, + $use_rgw = false, # ceph.conf Global settings - $auth_supported = 'cephx', - $osd_journal_size = '2048', - $osd_mkfs_type = 'xfs', - $osd_pool_default_size = '2', - $osd_pool_default_min_size = '1', + $auth_supported = 'cephx', + $osd_journal_size = '2048', + $osd_mkfs_type = 'xfs', + $osd_pool_default_size = '2', + $osd_pool_default_min_size = '1', # TODO: calculate PG numbers - $osd_pool_default_pg_num = '100', - $osd_pool_default_pgp_num = '100', - $cluster_network = $::fuel_settings['storage_network_range'], - $public_network = $::fuel_settings['management_network_range'], + $osd_pool_default_pg_num = '100', + $osd_pool_default_pgp_num = '100', + $cluster_network = $::fuel_settings['storage_network_range'], + $public_network = $::fuel_settings['management_network_range'], # RadosGW settings $rgw_host = $::fqdn, @@ -39,23 +37,23 @@ class ceph ( $rgw_nss_db_path = '/etc/ceph/nss', # Keystone settings - $rgw_pub_ip = $cluster_node_address, - $rgw_adm_ip = $cluster_node_address, - $rgw_int_ip = $cluster_node_address, + $rgw_pub_ip = $cluster_node_address, + $rgw_adm_ip = $cluster_node_address, + $rgw_int_ip = $cluster_node_address, # Cinder settings - $volume_driver = 'cinder.volume.drivers.rbd.RBDDriver', - $rbd_pool = 'volumes', - $glance_api_version = '2', - $rbd_user = 'volumes', + $volume_driver = 'cinder.volume.drivers.rbd.RBDDriver', + $glance_api_version = '2', + $cinder_user = 'volumes', + $cinder_pool = 'volumes', # TODO: generate rbd_secret_uuid - $rbd_secret_uuid = 'a5d0dd94-57c4-ae55-ffe0-7e3732a24455', + $rbd_secret_uuid = 'a5d0dd94-57c4-ae55-ffe0-7e3732a24455', # Glance settings - $glance_backend = 'ceph', - $rbd_store_user = 'images', - $rbd_store_pool = 'images', - $show_image_direct_url = 'True', + $glance_backend = 'ceph', + $glance_user = 'images', + $glance_pool = 'images', + $show_image_direct_url = 'True', ) { Exec { path => [ '/bin/', '/sbin/' , '/usr/bin/', '/usr/sbin/' ], @@ -80,8 +78,22 @@ class ceph ( case $::fuel_settings['role'] { 'primary-controller', 'controller', 'ceph-mon': { include ceph::mon - Class['ceph::conf'] -> - Class['ceph::mon'] -> + + # DO NOT SPLIT ceph auth command lines! See http://tracker.ceph.com/issues/3279 + ceph::pool {$glance_pool: + user => $glance_user, + acl => "mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=${glance_pool}'", + keyring_owner => 'glance', + } + + ceph::pool {$cinder_pool: + user => $cinder_user, + acl => "mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=${cinder_pool}, allow rx pool=${glance_pool}'", + keyring_owner => 'cinder', + } + + Class['ceph::conf'] -> Class['ceph::mon'] -> + Ceph::Pool[$glance_pool] -> Ceph::Pool[$cinder_pool] -> Service['ceph'] if ($::ceph::use_rgw) { diff --git a/deployment/puppet/ceph/manifests/mon.pp b/deployment/puppet/ceph/manifests/mon.pp index e1118b3b9d..3d97f45cbd 100644 --- a/deployment/puppet/ceph/manifests/mon.pp +++ b/deployment/puppet/ceph/manifests/mon.pp @@ -33,18 +33,8 @@ class ceph::mon { ], } - # creates the named OSD pool - define osd_pool { - exec { "Creating pool ${name}": - command => "ceph osd pool create ${name} ${::ceph::osd_pool_default_pg_num} ${::ceph::osd_pool_default_pgp_num}", - logoutput => true, - } - } - osd_pool {[$::ceph::cinder_pool, $::ceph::glance_pool]: } - Firewall['010 ceph-mon allow'] -> Exec['ceph-deploy mon create'] -> Exec['Wait for Ceph quorum'] -> - Exec['ceph-deploy gatherkeys'] -> - Osd_pool <||> + Exec['ceph-deploy gatherkeys'] } diff --git a/deployment/puppet/ceph/manifests/nova_compute.pp b/deployment/puppet/ceph/manifests/nova_compute.pp index 588ba4b12e..4145626ff3 100644 --- a/deployment/puppet/ceph/manifests/nova_compute.pp +++ b/deployment/puppet/ceph/manifests/nova_compute.pp @@ -1,6 +1,7 @@ # configure the nova_compute parts if present class ceph::nova_compute ( - $rbd_secret_uuid = $::ceph::rbd_secret_uuid + $rbd_secret_uuid = $::ceph::rbd_secret_uuid, + $user = $::ceph::cinder_user, ) { file {'/root/secret.xml': @@ -9,13 +10,11 @@ class ceph::nova_compute ( exec {'Set Ceph RBD secret for Nova': # TODO: clean this command up - command => 'virsh secret-set-value --secret $( \ + command => "virsh secret-set-value --secret $( \ virsh secret-define --file /root/secret.xml | \ - egrep -o "[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}") \ - --base64 $(ceph auth get-key client.volumes) && \ - rm /root/secret.xml', - require => File['/root/secret.xml'], - returns => [0,1], + egrep -o '[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}') \ + --base64 $(ceph auth get-key client.${user}) && \ + rm /root/secret.xml", } File['/root/secret.xml'] -> diff --git a/deployment/puppet/ceph/manifests/pool.pp b/deployment/puppet/ceph/manifests/pool.pp new file mode 100644 index 0000000000..5257a256f1 --- /dev/null +++ b/deployment/puppet/ceph/manifests/pool.pp @@ -0,0 +1,45 @@ +# create a Ceph pool with an associated Cephx user and ACL + +define ceph::pool ( + # Cephx user and ACL + $user = $name, + $acl = "mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=${name}'", + + # Unix user and group for the keyring file + $keyring_owner = $user, + $keyring_group = $keyring_owner, + + # Pool settings + $pg_num = $::ceph::osd_pool_default_pg_num, + $pgp_num = $::ceph::osd_pool_default_pgp_num, +) { + + exec {"Create ${name} pool": + command => "ceph osd pool create ${name} ${pg_num} ${pgp_num}", + unless => "rados lspools | grep -q '^${name}$'", + } + + exec {"Create ${user} Cephx user and ACL": + command => "ceph auth get-or-create client.${user} ${acl}", + unless => "ceph auth list | grep -q '^client.${user}$'" + } + + $keyring = "/etc/ceph/ceph.client.${user}.keyring" + + exec {"Populate ${user} keyring": + command => "ceph auth get-or-create client.${user} > ${keyring}", + creates => $keyring, + } + + file {$keyring: + ensure => file, + mode => '0640', + owner => $keyring_owner, + group => $keyring_group, + } + + Exec["Create ${name} pool"] -> + Exec["Create ${user} Cephx user and ACL"] -> + Exec["Populate ${user} keyring"] -> + File[$keyring] +} diff --git a/deployment/puppet/cinder/manifests/volume/ceph.pp b/deployment/puppet/cinder/manifests/volume/ceph.pp index 22a0cd67cb..85cb375f27 100644 --- a/deployment/puppet/cinder/manifests/volume/ceph.pp +++ b/deployment/puppet/cinder/manifests/volume/ceph.pp @@ -1,8 +1,9 @@ +# configures the Ceph RBD backend for Cinder class cinder::volume::ceph ( $volume_driver = $::ceph::volume_driver, - $rbd_pool = $::ceph::rbd_pool, $glance_api_version = $::ceph::glance_api_version, - $rbd_user = $::ceph::rbd_user, + $rbd_pool = $::ceph::cinder_pool, + $rbd_user = $::ceph::cinder_user, $rbd_secret_uuid = $::ceph::rbd_secret_uuid, ) { @@ -16,40 +17,18 @@ class cinder::volume::ceph ( File_line<||> ~> Service['cinder-volume'] # TODO: this needs to be re-worked to follow https://wiki.openstack.org/wiki/Cinder-multi-backend cinder_config { - 'DEFAULT/volume_driver': value => $volume_driver; - 'DEFAULT/rbd_pool': value => $rbd_pool; - 'DEFAULT/glance_api_version': value => $glance_api_version; - 'DEFAULT/rbd_user': value => $rbd_user; - 'DEFAULT/rbd_secret_uuid': value => $rbd_secret_uuid; + 'DEFAULT/volume_driver': value => $volume_driver; + 'DEFAULT/glance_api_version': value => $glance_api_version; + 'DEFAULT/rbd_pool': value => $rbd_pool; + 'DEFAULT/rbd_user': value => $rbd_user; + 'DEFAULT/rbd_secret_uuid': value => $rbd_secret_uuid; } + # TODO: convert to cinder params file {$::ceph::params::service_cinder_volume_opts: ensure => 'present', } -> file_line {'cinder-volume.conf': path => $::ceph::params::service_cinder_volume_opts, - line => "export CEPH_ARGS='--id ${::ceph::cinder_pool}'", + line => "export CEPH_ARGS='--id ${rbd_pool}'", } - - exec {'Create Cinder Ceph client ACL': - # DO NOT SPLIT ceph auth command lines! See http://tracker.ceph.com/issues/3279 - command => "ceph auth get-or-create client.${::ceph::cinder_pool} mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=${::ceph::cinder_pool}, allow rx pool=${::ceph::glance_pool}'", - logoutput => true, - } - - $cinder_keyring = "/etc/ceph/ceph.client.${::ceph::cinder_pool}.keyring" - exec {'Create keys for the Cinder pool': - command => "ceph auth get-or-create client.${::ceph::cinder_pool} > ${cinder_keyring}", - before => File[$cinder_keyring], - creates => $cinder_keyring, - require => Exec['Create Cinder Ceph client ACL'], - notify => Service['cinder-volume'], - returns => 0, - } - - file {$cinder_keyring: - owner => cinder, - group => cinder, - require => Exec['Create keys for the Cinder pool'], - mode => '0600', - } -} \ No newline at end of file +} diff --git a/deployment/puppet/glance/manifests/backend/ceph.pp b/deployment/puppet/glance/manifests/backend/ceph.pp index 2ce3f4188a..2be23cad37 100644 --- a/deployment/puppet/glance/manifests/backend/ceph.pp +++ b/deployment/puppet/glance/manifests/backend/ceph.pp @@ -1,8 +1,8 @@ -#configures the glance blacked for ceph (rbd) driver +# configures the Ceph RBD backend for Glance class glance::backend::ceph( $default_store = 'rbd', - $rbd_store_user = $::ceph::rbd_store_user, - $rbd_store_pool = $::ceph::rbd_store_pool, + $rbd_store_user = $::ceph::glance_user, + $rbd_store_pool = $::ceph::glance_pool, $show_image_direct_url = $::ceph::show_image_direct_url, ) inherits glance::api { @@ -17,32 +17,9 @@ class glance::backend::ceph( } glance_api_config { - 'DEFAULT/default_store': value => $default_store; - 'DEFAULT/rbd_store_user': value => $rbd_store_user; - 'DEFAULT/rbd_store_pool': value => $rbd_store_pool; - 'DEFAULT/show_image_direct_url': value => $show_image_direct_url; + 'DEFAULT/default_store': value => $default_store; + 'DEFAULT/rbd_store_user': value => $rbd_store_user; + 'DEFAULT/rbd_store_pool': value => $rbd_store_pool; + 'DEFAULT/show_image_direct_url': value => $show_image_direct_url; }~> Service['glance-api'] - - exec {'Create Glance Ceph client ACL': - # DO NOT SPLIT ceph auth command lines! See http://tracker.ceph.com/issues/3279 - command => "ceph auth get-or-create client.${::ceph::glance_pool} mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=${::ceph::glance_pool}'", - logoutput => true, - } - - $glance_keyring = "/etc/ceph/ceph.client.${::ceph::glance_pool}.keyring" - exec {'Create keys for the Glance pool': - command => "ceph auth get-or-create client.${::ceph::glance_pool} > ${$glance_keyring}", - before => File[$glance_keyring], - creates => $glance_keyring, - require => Exec['Create Glance Ceph client ACL'], - notify => Service['glance-api'], - returns => 0, - } - - file {$glance_keyring: - owner => glance, - group => glance, - require => Exec['Create keys for the Glance pool'], - mode => '0600', - } }