openstack/fuel-library
Code Issues Proposed changes

cephx key and ceph pool management fixes

Browse Source 
* consistently parametrize pools, Cephx users and ACLs
 * unscatter creation of Cinder and Glance pools into ceph::pool
 * use parameterized Cephx user when generating Nova secret
 * create Glance pool before Cinder ACL
This commit is contained in:
Dmitry Borodaenko 2013-10-11 00:21:14 -07:00
parent bc37934368
commit 0fdb733b3f
6 changed files with 112 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
deployment/puppet/ceph/manifests/init.pp
View File

@ -2,25 +2,23 @@
class ceph ( class ceph (
# General settings # General settings
$cluster_node_address = $::ipaddress, #This should be the cluster service address $cluster_node_address = $::ipaddress, #This should be the cluster service address
$primary_mon = $::hostname, #This should be the first controller $primary_mon = $::hostname, #This should be the first controller
$cinder_pool = 'volumes', $osd_devices = split($::osd_devices_list, ' '),
$glance_pool = 'images', $use_ssl = false,
$osd_devices = split($::osd_devices_list, ' '), $use_rgw = false,
$use_ssl = false,
$use_rgw = false,
# ceph.conf Global settings # ceph.conf Global settings
$auth_supported = 'cephx', $auth_supported = 'cephx',
$osd_journal_size = '2048', $osd_journal_size = '2048',
$osd_mkfs_type = 'xfs', $osd_mkfs_type = 'xfs',
$osd_pool_default_size = '2', $osd_pool_default_size = '2',
$osd_pool_default_min_size = '1', $osd_pool_default_min_size = '1',
# TODO: calculate PG numbers # TODO: calculate PG numbers
$osd_pool_default_pg_num = '100', $osd_pool_default_pg_num = '100',
$osd_pool_default_pgp_num = '100', $osd_pool_default_pgp_num = '100',
$cluster_network = $::fuel_settings['storage_network_range'], $cluster_network = $::fuel_settings['storage_network_range'],
$public_network = $::fuel_settings['management_network_range'], $public_network = $::fuel_settings['management_network_range'],
# RadosGW settings # RadosGW settings
$rgw_host = $::fqdn, $rgw_host = $::fqdn,
@ -39,23 +37,23 @@ class ceph (
$rgw_nss_db_path = '/etc/ceph/nss', $rgw_nss_db_path = '/etc/ceph/nss',
# Keystone settings # Keystone settings
$rgw_pub_ip = $cluster_node_address, $rgw_pub_ip = $cluster_node_address,
$rgw_adm_ip = $cluster_node_address, $rgw_adm_ip = $cluster_node_address,
$rgw_int_ip = $cluster_node_address, $rgw_int_ip = $cluster_node_address,
# Cinder settings # Cinder settings
$volume_driver = 'cinder.volume.drivers.rbd.RBDDriver', $volume_driver = 'cinder.volume.drivers.rbd.RBDDriver',
$rbd_pool = 'volumes', $glance_api_version = '2',
$glance_api_version = '2', $cinder_user = 'volumes',
$rbd_user = 'volumes', $cinder_pool = 'volumes',
# TODO: generate rbd_secret_uuid # TODO: generate rbd_secret_uuid
$rbd_secret_uuid = 'a5d0dd94-57c4-ae55-ffe0-7e3732a24455', $rbd_secret_uuid = 'a5d0dd94-57c4-ae55-ffe0-7e3732a24455',
# Glance settings # Glance settings
$glance_backend = 'ceph', $glance_backend = 'ceph',
$rbd_store_user = 'images', $glance_user = 'images',
$rbd_store_pool = 'images', $glance_pool = 'images',
$show_image_direct_url = 'True', $show_image_direct_url = 'True',
) { ) {
Exec { path => [ '/bin/', '/sbin/' , '/usr/bin/', '/usr/sbin/' ], Exec { path => [ '/bin/', '/sbin/' , '/usr/bin/', '/usr/sbin/' ],
@ -80,8 +78,22 @@ class ceph (
case $::fuel_settings['role'] { case $::fuel_settings['role'] {
'primary-controller', 'controller', 'ceph-mon': { 'primary-controller', 'controller', 'ceph-mon': {
include ceph::mon include ceph::mon
Class['ceph::conf'] ->
Class['ceph::mon'] -> # DO NOT SPLIT ceph auth command lines! See http://tracker.ceph.com/issues/3279
ceph::pool {$glance_pool:
user => $glance_user,
acl => "mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=${glance_pool}'",
keyring_owner => 'glance',
}
ceph::pool {$cinder_pool:
user => $cinder_user,
acl => "mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=${cinder_pool}, allow rx pool=${glance_pool}'",
keyring_owner => 'cinder',
}
Class['ceph::conf'] -> Class['ceph::mon'] ->
Ceph::Pool[$glance_pool] -> Ceph::Pool[$cinder_pool] ->
Service['ceph'] Service['ceph']
if ($::ceph::use_rgw) { if ($::ceph::use_rgw) {

12
deployment/puppet/ceph/manifests/mon.pp
View File

@ -33,18 +33,8 @@ class ceph::mon {
], ],
} }
# creates the named OSD pool
define osd_pool {
exec { "Creating pool ${name}":
command => "ceph osd pool create ${name} ${::ceph::osd_pool_default_pg_num} ${::ceph::osd_pool_default_pgp_num}",
logoutput => true,
}
}
osd_pool {[$::ceph::cinder_pool, $::ceph::glance_pool]: }
Firewall['010 ceph-mon allow'] -> Firewall['010 ceph-mon allow'] ->
Exec['ceph-deploy mon create'] -> Exec['ceph-deploy mon create'] ->
Exec['Wait for Ceph quorum'] -> Exec['Wait for Ceph quorum'] ->
Exec['ceph-deploy gatherkeys'] -> Exec['ceph-deploy gatherkeys']
Osd_pool <||>
} }

13
deployment/puppet/ceph/manifests/nova_compute.pp
View File

@ -1,6 +1,7 @@
# configure the nova_compute parts if present # configure the nova_compute parts if present
class ceph::nova_compute ( class ceph::nova_compute (
$rbd_secret_uuid = $::ceph::rbd_secret_uuid $rbd_secret_uuid = $::ceph::rbd_secret_uuid,
$user = $::ceph::cinder_user,
) { ) {
file {'/root/secret.xml': file {'/root/secret.xml':
@ -9,13 +10,11 @@ class ceph::nova_compute (
exec {'Set Ceph RBD secret for Nova': exec {'Set Ceph RBD secret for Nova':
# TODO: clean this command up # TODO: clean this command up
command => 'virsh secret-set-value --secret $( \ command => "virsh secret-set-value --secret $( \
virsh secret-define --file /root/secret.xml | \ virsh secret-define --file /root/secret.xml | \
egrep -o "[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}") \ egrep -o '[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}') \
--base64 $(ceph auth get-key client.volumes) && \ --base64 $(ceph auth get-key client.${user}) && \
rm /root/secret.xml', rm /root/secret.xml",
require => File['/root/secret.xml'],
returns => [0,1],
} }
File['/root/secret.xml'] -> File['/root/secret.xml'] ->

45
deployment/puppet/ceph/manifests/pool.pp Normal file
View File

@ -0,0 +1,45 @@
# create a Ceph pool with an associated Cephx user and ACL
define ceph::pool (
# Cephx user and ACL
$user = $name,
$acl = "mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=${name}'",
# Unix user and group for the keyring file
$keyring_owner = $user,
$keyring_group = $keyring_owner,
# Pool settings
$pg_num = $::ceph::osd_pool_default_pg_num,
$pgp_num = $::ceph::osd_pool_default_pgp_num,
) {
exec {"Create ${name} pool":
command => "ceph osd pool create ${name} ${pg_num} ${pgp_num}",
unless => "rados lspools | grep -q '^${name}$'",
}
exec {"Create ${user} Cephx user and ACL":
command => "ceph auth get-or-create client.${user} ${acl}",
unless => "ceph auth list | grep -q '^client.${user}$'"
}
$keyring = "/etc/ceph/ceph.client.${user}.keyring"
exec {"Populate ${user} keyring":
command => "ceph auth get-or-create client.${user} > ${keyring}",
creates => $keyring,
}
file {$keyring:
ensure => file,
mode => '0640',
owner => $keyring_owner,
group => $keyring_group,
}
Exec["Create ${name} pool"] ->
Exec["Create ${user} Cephx user and ACL"] ->
Exec["Populate ${user} keyring"] ->
File[$keyring]
}

43
deployment/puppet/cinder/manifests/volume/ceph.pp
View File

@ -1,8 +1,9 @@
# configures the Ceph RBD backend for Cinder
class cinder::volume::ceph ( class cinder::volume::ceph (
$volume_driver = $::ceph::volume_driver, $volume_driver = $::ceph::volume_driver,
$rbd_pool = $::ceph::rbd_pool,
$glance_api_version = $::ceph::glance_api_version, $glance_api_version = $::ceph::glance_api_version,
$rbd_user = $::ceph::rbd_user, $rbd_pool = $::ceph::cinder_pool,
$rbd_user = $::ceph::cinder_user,
$rbd_secret_uuid = $::ceph::rbd_secret_uuid, $rbd_secret_uuid = $::ceph::rbd_secret_uuid,
) { ) {
@ -16,40 +17,18 @@ class cinder::volume::ceph (
File_line<||> ~> Service['cinder-volume'] File_line<||> ~> Service['cinder-volume']
# TODO: this needs to be re-worked to follow https://wiki.openstack.org/wiki/Cinder-multi-backend # TODO: this needs to be re-worked to follow https://wiki.openstack.org/wiki/Cinder-multi-backend
cinder_config { cinder_config {
'DEFAULT/volume_driver': value => $volume_driver; 'DEFAULT/volume_driver': value => $volume_driver;
'DEFAULT/rbd_pool': value => $rbd_pool; 'DEFAULT/glance_api_version': value => $glance_api_version;
'DEFAULT/glance_api_version': value => $glance_api_version; 'DEFAULT/rbd_pool': value => $rbd_pool;
'DEFAULT/rbd_user': value => $rbd_user; 'DEFAULT/rbd_user': value => $rbd_user;
'DEFAULT/rbd_secret_uuid': value => $rbd_secret_uuid; 'DEFAULT/rbd_secret_uuid': value => $rbd_secret_uuid;
} }
# TODO: convert to cinder params # TODO: convert to cinder params
file {$::ceph::params::service_cinder_volume_opts: file {$::ceph::params::service_cinder_volume_opts:
ensure => 'present', ensure => 'present',
} -> file_line {'cinder-volume.conf': } -> file_line {'cinder-volume.conf':
path => $::ceph::params::service_cinder_volume_opts, path => $::ceph::params::service_cinder_volume_opts,
line => "export CEPH_ARGS='--id ${::ceph::cinder_pool}'", line => "export CEPH_ARGS='--id ${rbd_pool}'",
} }
}
exec {'Create Cinder Ceph client ACL':
# DO NOT SPLIT ceph auth command lines! See http://tracker.ceph.com/issues/3279
command => "ceph auth get-or-create client.${::ceph::cinder_pool} mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=${::ceph::cinder_pool}, allow rx pool=${::ceph::glance_pool}'",
logoutput => true,
}
$cinder_keyring = "/etc/ceph/ceph.client.${::ceph::cinder_pool}.keyring"
exec {'Create keys for the Cinder pool':
command => "ceph auth get-or-create client.${::ceph::cinder_pool} > ${cinder_keyring}",
before => File[$cinder_keyring],
creates => $cinder_keyring,
require => Exec['Create Cinder Ceph client ACL'],
notify => Service['cinder-volume'],
returns => 0,
}
file {$cinder_keyring:
owner => cinder,
group => cinder,
require => Exec['Create keys for the Cinder pool'],
mode => '0600',
}
}

37
deployment/puppet/glance/manifests/backend/ceph.pp
View File

@ -1,8 +1,8 @@
#configures the glance blacked for ceph (rbd) driver # configures the Ceph RBD backend for Glance
class glance::backend::ceph( class glance::backend::ceph(
$default_store = 'rbd', $default_store = 'rbd',
$rbd_store_user = $::ceph::rbd_store_user, $rbd_store_user = $::ceph::glance_user,
$rbd_store_pool = $::ceph::rbd_store_pool, $rbd_store_pool = $::ceph::glance_pool,
$show_image_direct_url = $::ceph::show_image_direct_url, $show_image_direct_url = $::ceph::show_image_direct_url,
) inherits glance::api { ) inherits glance::api {
@ -17,32 +17,9 @@ class glance::backend::ceph(
} }
glance_api_config { glance_api_config {
'DEFAULT/default_store': value => $default_store; 'DEFAULT/default_store': value => $default_store;
'DEFAULT/rbd_store_user': value => $rbd_store_user; 'DEFAULT/rbd_store_user': value => $rbd_store_user;
'DEFAULT/rbd_store_pool': value => $rbd_store_pool; 'DEFAULT/rbd_store_pool': value => $rbd_store_pool;
'DEFAULT/show_image_direct_url': value => $show_image_direct_url; 'DEFAULT/show_image_direct_url': value => $show_image_direct_url;
}~> Service['glance-api'] }~> Service['glance-api']
exec {'Create Glance Ceph client ACL':
# DO NOT SPLIT ceph auth command lines! See http://tracker.ceph.com/issues/3279
command => "ceph auth get-or-create client.${::ceph::glance_pool} mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=${::ceph::glance_pool}'",
logoutput => true,
}
$glance_keyring = "/etc/ceph/ceph.client.${::ceph::glance_pool}.keyring"
exec {'Create keys for the Glance pool':
command => "ceph auth get-or-create client.${::ceph::glance_pool} > ${$glance_keyring}",
before => File[$glance_keyring],
creates => $glance_keyring,
require => Exec['Create Glance Ceph client ACL'],
notify => Service['glance-api'],
returns => 0,
}
file {$glance_keyring:
owner => glance,
group => glance,
require => Exec['Create keys for the Glance pool'],
mode => '0600',
}
} }