openstack/fuel-library
Code Issues Proposed changes

RadosGW for CEPH

Browse Source
This commit is contained in:
Alexander Noskov 2013-08-22 05:21:31 -04:00 committed by Andrew Woodward
parent 087c323361
commit 719303e934
8 changed files with 244 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
deployment/puppet/ceph/examples/site.pp
View File

@ -4,16 +4,19 @@ Exec { path => [ '/bin/', '/sbin/' , '/usr/bin/', '/usr/sbin/' ] }
# This parameters defines nodes for CEPH cluster.
# The last node in this case is the master node of CEPH cluster and should be deployed last.
$nodes = [
'fuel-ceph-01.local.try',
'fuel-ceph-02.local.try'
'fuel-controller-02.local.try',
'fuel-controller-03.local.try',
]
# Uncomment this line if you want to install RadosGW.
$rados_GW = 'fuel-controller-03.local.try'
# Uncomment this line if you want to install metadata server.
# $mds_server = 'fuel-ceph-02.local.try'
$mds_server = 'fuel-controller-03.local.try'
# This parameter defines which devices to aggregate into CEPH cluster.
# ALL THE DATA THAT RESIDES ON THESE DEVICES WILL BE LOST!
$osd_devices = [ 'vdb', 'vdc' ]
$osd_devices = [ 'sdb', 'sdc' ]
# This parameter defines rbd pools for Cinder & Glance. It is not necessary to change.
$pools = [ 'volumes', 'images' ]
@ -30,28 +33,59 @@ node 'default' {
ensure => latest,
}
if $fqdn == $nodes[-1] and !str2bool($::ceph_conf) {
class { 'ceph::deploy':
auth_supported => 'cephx',
osd_journal_size => '2048',
osd_mkfs_type => 'xfs',
}
package {['ceph-deploy']:
ensure => latest,
require => Class['apt::update']
}
class { 'ceph::deploy':
#Global settings
auth_supported => 'cephx',
osd_journal_size => '2048',
osd_mkfs_type => 'xfs',
osd_pool_default_size => '2',
osd_pool_default_min_size => '1',
osd_pool_default_pg_num => '100',
osd_pool_default_pgp_num => '100',
cluster_network => '10.0.0.0/24',
public_network => '192.168.0.0/24',
#RadosGW settings
host => $::hostname,
keyring_path => '/etc/ceph/keyring.radosgw.gateway',
rgw_socket_path => '/tmp/radosgw.sock',
log_file => '/var/log/ceph/radosgw.log',
user => 'www-data',
rgw_keystone_url => '10.0.0.223:5000',
rgw_keystone_admin_token => 'nova',
rgw_keystone_token_cache_size => '10',
rgw_keystone_revocation_interval => '60',
rgw_data => '/var/lib/ceph/rados',
rgw_dns_name => $::hostname,
rgw_print_continue => 'false',
nss_db_path => '/etc/ceph/nss',
}
}
if $fqdn == $rados_GW {
ceph::radosgw {"${::hostname}":
require => Class['apt::update', 'ceph::deploy']
}
}
class { 'ceph::glance':
default_store => 'rbd',
rbd_store_user => 'images',
rbd_store_pool => 'images',
show_image_direct_url => 'True'
show_image_direct_url => 'True',
}
class { 'ceph::cinder':
volume_driver => 'cinder.volume.drivers.rbd.RBDDriver',
rbd_pool => 'volumes',
glance_api_version => '2',
rbd_user => 'volumes',
rbd_secret_uuid => 'a5d0dd94-57c4-ae55-ffe0-7e3732a24455'
rbd_secret_uuid => 'a5d0dd94-57c4-ae55-ffe0-7e3732a24455',
}
class { 'ceph::nova_compute': }
ceph::keystone { "Keystone":
pub_ip => "${rados_GW}",
adm_ip => "${rados_GW}",
int_ip => "${rados_GW}",
}
}

9
deployment/puppet/ceph/lib/facter/keystone_conf.rb Normal file
View File

@ -0,0 +1,9 @@
Facter.add("keystone_conf") do
setcode do
File.exists? '/etc/keystone/keystone.conf'
end
end

61
deployment/puppet/ceph/manifests/deploy.pp
View File

@ -1,7 +1,26 @@
class ceph::deploy (
$auth_supported,
$osd_journal_size,
$osd_mkfs_type,
$auth_supported = 'cephx',
$osd_journal_size = '2048',
$osd_mkfs_type = 'xfs',
$osd_pool_default_size = '2',
$osd_pool_default_min_size = '0',
$osd_pool_default_pg_num = '8',
$osd_pool_default_pgp_num = '8',
$cluster_network = '10.0.0.0/24',
$public_network = '192.168.0.0/24',
$host = $hostname,
$keyring_path = '/etc/ceph/keyring.radosgw.gateway',
$rgw_socket_path = '/tmp/radosgw.sock',
$log_file = '/var/log/ceph/radosgw.log',
$user = 'www-data',
$rgw_keystone_url = '127.0.0.1:5000',
$rgw_keystone_admin_token = 'nova',
$rgw_keystone_token_cache_size = '10',
$rgw_keystone_revocation_interval = '60',
$rgw_data = '/var/lib/ceph/rados',
$rgw_dns_name = $hostname,
$rgw_print_continue = 'false',
$nss_db_path = '/etc/ceph/nss',
) {
include p_osd, c_osd, c_pools
@ -10,14 +29,36 @@ class ceph::deploy (
command => "ceph-deploy new ${range}",
require => Package['ceph-deploy', 'ceph']
}
Ceph_conf {require => Exec['ceph-deploy-s1']}
ceph_conf {
'global/auth supported': value => $auth_supported, require => Exec['ceph-deploy-s1'];
'global/osd journal size': value => $osd_journal_size, require => Exec['ceph-deploy-s1'];
'global/osd mkfs type': value => $osd_mkfs_type, require => Exec['ceph-deploy-s1'];
'global/auth supported': value => $auth_supported;
'global/osd journal size': value => $osd_journal_size;
'global/osd mkfs type': value => $osd_mkfs_type;
'global/osd pool default size': value => $osd_pool_default_size;
'global/osd pool default min size': value => $osd_pool_default_min_size;
'global/osd pool default pg num': value => $osd_pool_default_pg_num;
'global/osd pool default pgp num': value => $osd_pool_default_pgp_num;
'global/cluster network': value => $cluster_network;
'global/public network': value => $public_network;
'client.radosgw.gateway/host': value => $host;
'client.radosgw.gateway/keyring': value => $keyring_path;
'client.radosgw.gateway/rgw socket path': value => $rgw_socket_path;
'client.radosgw.gateway/log file': value => $log_file;
'client.radosgw.gateway/user': value => $user;
'client.radosgw.gateway/rgw keystone url': value => $rgw_keystone_url;
'client.radosgw.gateway/rgw keystone admin token': value => $rgw_keystone_admin_token;
'client.radosgw.gateway/rgw keystone accepted roles': value => $rgw_keystone_accepted_roles;
'client.radosgw.gateway/rgw keystone token cache size': value => $rgw_keystone_token_cache_size;
'client.radosgw.gateway/rgw keystone revocation interval': value => $rgw_keystone_revocation_interval;
'client.radosgw.gateway/rgw data': value => $rgw_data;
'client.radosgw.gateway/rgw dns name': value => $rgw_dns_name;
'client.radosgw.gateway/rgw print continue': value => $rgw_print_continue;
'client.radosgw.gateway/nss db path': value => $nss_db_path;
}
Ceph_conf <||> -> Exec ['ceph-deploy-s2']
exec { 'ceph-deploy-s2':
command => "ceph-deploy --overwrite-conf mon create ${range}",
require => Ceph_conf['global/auth supported', 'global/osd journal size', 'global/osd mkfs type']
# require => Ceph_conf['global/auth supported', 'global/osd journal size', 'global/osd mkfs type']
}
File {
ensure => 'link',
@ -54,16 +95,16 @@ class ceph::deploy (
}
int { $osd_devices: }
}
if $mds {
if $mds_server {
exec { 'ceph-deploy-s4':
command => "ceph-deploy mds create ${mds}",
command => "ceph-deploy mds create ${mds_server}",
require => Class['c_osd']
}
}
class c_pools {
define int {
exec { "Creating pool ${name}":
command => "ceph osd pool create ${name} 128",
command => "ceph osd pool create ${name} ${osd_pool_default_pg_num} ${osd_pool_default_pgp_num}",
require => Class['c_osd']
}
}

1
deployment/puppet/ceph/manifests/glance.pp
View File

@ -31,6 +31,7 @@ class ceph::glance (
command => 'ceph auth get-or-create client.images > /etc/ceph/ceph.client.images.keyring',
before => File['/etc/ceph/ceph.client.images.keyring'],
require => [Package['ceph'], Exec['Copy config']],
notify => Service['glance-api'],
returns => [0,1],
}
file { '/etc/ceph/ceph.client.images.keyring':

43
deployment/puppet/ceph/manifests/keystone.pp Normal file
View File

@ -0,0 +1,43 @@
define ceph::keystone (
$pub_ip,
$adm_ip,
$int_ip,
$directory = '/etc/ceph/nss',
) {
if str2bool($::keystone_conf) {
package { "libnss3-tools" :
ensure => 'latest'
}
file { "${directory}":
ensure => "directory",
require => Package['ceph'],
}
exec {"creating OpenSSL certificates":
command => "openssl x509 -in /etc/keystone/ssl/certs/ca.pem -pubkey \
| certutil -d ${directory} -A -n ca -t 'TCu,Cu,Tuw' && openssl x509 \
-in /etc/keystone/ssl/certs/signing_cert.pem -pubkey | certutil -A -d \
${directory} -n signing_cert -t 'P,P,P'",
require => [File["${directory}"], Package["libnss3-tools"]]
} ->
exec {"copy OpenSSL certificates":
command => "scp -r /etc/ceph/nss/* ${rados_GW}:/etc/ceph/nss/ && ssh ${rados_GW} '/etc/init.d/radosgw restart'",
}
keystone_service { "swift":
ensure => present,
type => 'object-store',
description => 'Openstack Object-Store Service',
notify => Service['keystone'],
}
keystone_endpoint { "RegionOne/swift":
ensure => present,
public_url => "http://${pub_ip}/swift/v1",
admin_url => "http://${adm_ip}/swift/v1",
internal_url => "http://${int_ip}/swift/v1",
notify => Service['keystone'],
}
service { "keystone":
enable => true,
ensure => "running",
}
}
}

70
deployment/puppet/ceph/manifests/radosgw.pp Normal file
View File

@ -0,0 +1,70 @@
define apache::loadmodule () {
exec { "/usr/sbin/a2enmod $name" :
unless => "/bin/readlink -e /etc/apache2/mods-enabled/${name}.load",
notify => Service[apache2]
}
}
define ceph::radosgw (
$keyring_path = '/etc/ceph/keyring.radosgw.gateway',
$apache2_ssl = '/etc/apache2/ssl/',
$radosgw_auth_key = 'client.radosgw.gateway',
) {
package { ["apache2", "libapache2-mod-fastcgi", 'libnss3-tools', 'radosgw']:
ensure => "latest",
}
apache::loadmodule{["rewrite", "fastcgi", "ssl"]: }
file {'/etc/apache2/httpd.conf':
ensure => "present",
content => "ServerName ${fqdn}",
notify => Service["apache2"],
require => Package["apache2"],
}
file {["${apache2_ssl}", '/var/lib/ceph/radosgw/ceph-radosgw.gateway', '/var/lib/ceph/radosgw', '/etc/ceph/nss']:
ensure => "directory",
mode => 755,
}
exec {"generate SSL certificate on $name":
command => "openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ${apache2_ssl}apache.key -out ${apache2_ssl}apache.crt -subj '/C=RU/ST=Russia/L=Saratov/O=Mirantis/OU=CA/CN=localhost'",
returns => [0,1],
}
file { "/etc/apache2/sites-available/rgw.conf":
content => template('ceph/rgw.conf.erb'),
notify => Service["apache2"],
require => Package["apache2"],
}
Exec {require => File["/etc/apache2/sites-available/rgw.conf"]}
exec {'a2ensite rgw.conf':}
exec {'a2dissite default':}
file { "/var/www/s3gw.fcgi":
content => template('ceph/s3gw.fcgi.erb'),
notify => Service["apache2"],
require => Package["apache2"],
mode => "+x",
}
exec { "ceph-create-radosgw-keyring-on $name":
command => "ceph-authtool --create-keyring ${keyring_path}",
require => Package['ceph'],
} ->
file { "${keyring_path}":
mode => "+r",
} ->
exec { "ceph-generate-key-on $name":
command => "ceph-authtool ${keyring_path} -n ${radosgw_auth_key} --gen-key",
require => Package["apache2"],
} ->
exec { "ceph-add-capabilities-to-the-key-on $name":
command => "ceph-authtool -n ${radosgw_auth_key} --cap osd 'allow rwx' --cap mon 'allow rw' ${keyring_path}",
require => Package["apache2"],
} ->
exec { "ceph-add-to-ceph-keyring-entries-on $name":
command => "ceph -k /etc/ceph/ceph.client.admin.keyring auth add ${radosgw_auth_key} -i ${keyring_path}",
require => Package["apache2"],
}
service { "apache2":
enable => true,
ensure => "running",
}
}

23
deployment/puppet/ceph/templates/rgw.conf.erb Normal file
View File

@ -0,0 +1,23 @@
FastCgiExternalServer /var/www/s3gw.fcgi -socket /tmp/radosgw.sock
<VirtualHost *:80>
ServerName <%= @fqdn %>
DocumentRoot /var/www
RewriteEngine On
RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) /s3gw.fcgi?page=$1&params=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
<IfModule mod_fastcgi.c>
<Directory /var/www>
Options +ExecCGI
AllowOverride All
SetHandler fastcgi-script
Order allow,deny
Allow from all
AuthBasicAuthoritative Off
</Directory>
</IfModule>
AllowEncodedSlashes On
ServerSignature Off
</VirtualHost>

2
deployment/puppet/ceph/templates/s3gw.fcgi.erb Normal file
View File

@ -0,0 +1,2 @@
#!/bin/sh
exec /usr/bin/radosgw -c /etc/ceph/ceph.conf -n client.radosgw.gateway