From 592c7d57115711ee13f387e1ebe7815ef7188eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 12 Jan 2016 13:22:29 +0000 Subject: [PATCH] Add radosgw (aka rgw) support This patch enables the plugin to setup ceph radosgw and configures it as a swift-compatible endpoint with keystone integration. ENABLE_CEPH_RGW is the new variable introduced, which is False by default, and can be set to True in localrc if radosgw needs to be setup. Fixed couple of other (related) things ... 1) Created rgw specific functions 2) Checking for radosgw & swift co-existence, erroring out early enough NOTE: Tested on ubuntu trusty only TODOs ... 1) Not tested on Fedora (F22), radosgw startup has some issues there. Co-Authored-By: Deepak C Shetty Change-Id: I21db4168eb69d107599f6b6ab1668b02b764b2c6 --- README.md | 12 ++++- devstack/lib/ceph | 91 +++++++++++++++++++++++--------------- devstack/override-defaults | 2 + devstack/plugin.sh | 20 ++++----- 4 files changed, 77 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 19dd812b..fa44d28b 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ As part of ```stack.sh```: * Creates a Ceph cluster for use with openstack services * Configures Ceph as the storage backend for Cinder, Cinder Backup, Nova, Manila (not by default), and Glance services +* (Optionally) Sets up & configures Rados gateway (aka rgw or radosgw) as a Swift endpoint with Keystone integration + * Set ```ENABLE_CEPH_RGW=True``` in your ```localrc``` * Supports Ceph cluster running local or remote to openstack services As part of ```unstack.sh``` | ```clean.sh```: @@ -50,9 +52,17 @@ This plugin also gets used to configure Ceph as the storage backend for the upst * Then run ```stack.sh``` and wait for the _magic_ to happen :) +# Known Issues / Limitations + +* Rados Gateway with Keystone for Swift - works on Ubuntu only +* Tempest test failures when using RGW as swift endpoint +* Tempest fails due to verify-tempest-config erroring out, when using RGW as swift endpoint + * Patch sent @ https://review.openstack.org/#/c/264179/ + + # TODOs -* Configuring Rados Gateway with Keystone for Swift +* Fix Rados Gateway with Keystone for Swift on Fedora * Add support for Ceph Infernalis release * Add support for distro specific ceph repos * Add Manila support for non-Ubuntu systems diff --git a/devstack/lib/ceph b/devstack/lib/ceph index 790cd8f4..08f825a9 100644 --- a/devstack/lib/ceph +++ b/devstack/lib/ceph @@ -217,7 +217,10 @@ fi } function cleanup_ceph_embedded { -sudo killall -w -9 ceph-mon ceph-osd radosgw +sudo killall -w -9 ceph-mon ceph-osd +if [ "$ENABLE_CEPH_RGW" = "True" ]; then + sudo killall -w -9 radosgw +fi sudo rm -rf ${CEPH_DATA_DIR}/*/* if egrep -q ${CEPH_DATA_DIR} /proc/mounts; then sudo umount ${CEPH_DATA_DIR} @@ -361,17 +364,21 @@ if is_ceph_enabled_for_service manila; then fi fi +if [ "$ENABLE_CEPH_RGW" = "True" ]; then + configure_ceph_rgw +fi +} + +function configure_ceph_rgw { # bootstrap rados gateway -sudo mkdir ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname) +sudo mkdir -p ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname) sudo ceph auth get-or-create client.radosgw.$(hostname) \ osd 'allow rwx' mon 'allow rw' \ --o /etc/ceph/ceph.client.radosgw.$(hostname).keyring - -sudo cp /etc/ceph/ceph.client.radosgw.$(hostname).keyring \ -${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/keyring +-o ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/keyring if is_ubuntu; then - sudo touch ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}/{upstart,done} + sudo touch \ + ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/{upstart,done} else sudo touch \ ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/{sysvinit,done} @@ -382,6 +389,24 @@ fi if [[ $(echo $(get_ceph_version mon) '>=' 9.2 | bc -l) == 1 ]]; then sudo chown -R ceph. ${CEPH_DATA_DIR} fi + +if [[ ! "$(egrep "\[client.radosgw\]" ${CEPH_CONF_FILE})" ]]; then + cat </dev/null +[client.radosgw.$(hostname)] +host = $(hostname) +keyring = ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/keyring +rgw socket path = /tmp/radosgw-$(hostname).sock +log file = /var/log/ceph/radosgw-$(hostname).log +rgw data = ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname) +rgw print continue = false +rgw frontends = civetweb port=${CEPH_RGW_PORT} +rgw keystone url = http://${SERVICE_HOST}:35357 +rgw keystone admin token = ${SERVICE_TOKEN} +rgw keystone accepted roles = Member, _member_, admin +rgw s3 auth use keystone = true +nss db path = ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss +EOF +fi } function configure_ceph_embedded_rgw { @@ -398,23 +423,8 @@ if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:${CEPH_RGW_PORT}/swift/v1" fi -if [[ ! "$(egrep "\[client.radosgw\]" ${CEPH_CONF_FILE})" ]]; then - cat </dev/null -[client.radosgw.$(hostname)] -host = $(hostname) -keyring = /var/lib/ceph/radosgw/ceph-radosgw.$(hostname)/keyring -rgw socket path = /tmp/radosgw-$(hostname).sock -log file = /var/log/ceph/radosgw-$(hostname).log -rgw data = /var/lib/ceph/radosgw/ceph-radosgw.$(hostname) -rgw print continue = false -rgw frontends = civetweb port=${CEPH_RGW_PORT} -rgw keystone url = http://${SERVICE_HOST}:35357 -rgw keystone admin token = ${SERVICE_TOKEN} -rgw keystone accepted roles = Member, _member_, admin -rgw s3 auth use keystone = true -nss db path = ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss -EOF -fi +# Let keystone generate the certs, rgw needs these. +keystone-manage pki_setup --rebuild # radosgw needs to access keystone's revocation list sudo mkdir ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss @@ -427,9 +437,13 @@ sudo openssl x509 -in /etc/keystone/ssl/certs/signing_cert.pem -pubkey | \ sudo certutil -A \ -d ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss \ -n signing_cert -t "P,P,P" + +# radosgw service is started here as it needs the keystone pki_setup as a +# pre-requisite +sudo start radosgw id=radosgw.$(hostname) } -function configure_ceph_remote_radosgw { +function configure_ceph_remote_rgw { if [[ -z "$CEPH_REMOTE_RGW_URL" ]]; then die $LINENO \ "You activated REMOTE_CEPH_RGW thus CEPH_REMOTE_RGW_URL must be defined" @@ -583,7 +597,9 @@ function init_ceph { # make sure to kill all ceph processes first sudo pkill -f ceph-mon || true sudo pkill -f ceph-osd || true -sudo pkill -f radosgw || true +if [ "$ENABLE_CEPH_RGW" = "True" ]; then + sudo pkill -f radosgw || true +fi if is_ceph_enabled_for_service manila; then sudo pkill -f ceph-mds || true fi @@ -608,7 +624,7 @@ if is_ubuntu; then # Update package repo. REPOS_UPDATED=False - install_package ceph ceph-mds radosgw libnss3-tools + install_package ceph ceph-mds libnss3-tools else wget -q -O- 'https://download.ceph.com/keys/release.asc' \ | sudo apt-key add - @@ -618,15 +634,21 @@ if is_ubuntu; then # Update package repo. REPOS_UPDATED=False - install_package ceph radosgw libnss3-tools + install_package ceph libnss3-tools fi + if [ "$ENABLE_CEPH_RGW" = "True" ]; then + install_package radosgw + fi else # Install directly from distro repos. See LP bug 1521073 for more details. # If distro doesn't carry latest ceph, users can install latest ceph repo # for their distro (if available) from download.ceph.com and then do # stack.sh - install_package ceph ceph-radosgw + install_package ceph + if [ "$ENABLE_CEPH_RGW" = "True" ]; then + install_package ceph-radosgw + fi fi } @@ -648,23 +670,22 @@ if is_ubuntu; then else sudo service ceph start fi - -# FIXME: Some issues with radosgw start, disabling it for now -#sudo service radosgw start } # stop_ceph() - Stop running processes (non-screen) function stop_ceph { if is_ubuntu; then - sudo service ceph-mon-all stop > /dev/null 2>&1 - sudo service ceph-osd-all stop > /dev/null 2>&1 + sudo stop ceph-mon-all > /dev/null 2>&1 + sudo stop ceph-osd-all > /dev/null 2>&1 + if [ "$ENABLE_CEPH_RGW" = "True" ]; then + sudo stop radosgw-all > /dev/null 2>&1 + fi if is_ceph_enabled_for_service manila; then sudo service ceph-mds-all stop > /dev/null 2>&1 fi else sudo service ceph stop > /dev/null 2>&1 fi -sudo service radosgw stop > /dev/null 2>&1 } diff --git a/devstack/override-defaults b/devstack/override-defaults index 9df55dec..73d7f6b6 100644 --- a/devstack/override-defaults +++ b/devstack/override-defaults @@ -7,6 +7,8 @@ ENABLE_CEPH_GLANCE=$(trueorfalse True ENABLE_CEPH_GLANCE) # CephFS Manila driver is WIP. ENABLE_CEPH_MANILA=$(trueorfalse False ENABLE_CEPH_MANILA) ENABLE_CEPH_NOVA=$(trueorfalse True ENABLE_CEPH_NOVA) +# Do not enable RGW by default as RGW is not tested in upstream CI. +ENABLE_CEPH_RGW=$(trueorfalse False ENABLE_CEPH_RGW) if [[ $ENABLE_CEPH_CINDER == "True" ]]; then CINDER_DRIVER=ceph diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 7aaf7dcc..8088d305 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -4,6 +4,11 @@ if [[ "$1" == "source" ]]; then # Initial source source $TOP_DIR/lib/ceph elif [[ "$1" == "stack" && "$2" == "pre-install" ]]; then + if [[ "$ENABLE_CEPH_RGW" = "True" ]] && (is_service_enabled swift); then + die $LINENO \ + "You cannot activate both Swift and Ceph Rados Gateway, \ + please disable Swift or set ENABLE_CEPH_RGW=False" + fi echo_summary "Installing Ceph" check_os_support_ceph if [ "$REMOTE_CEPH" = "False" ]; then @@ -62,18 +67,9 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then echo_summary "Configuring Manila for Ceph" configure_ceph_embedded_manila fi - # FIXME: Fix this once radosgw service is running - - #echo_summary "Configuring Rados Gateway with Keystone for Swift" - #configure_ceph_embedded_rgw - fi - if [ "$REMOTE_CEPH_RGW" = "True" ]; then - if is_service_enabled swift; then - die $LINENO \ - "You can not activate both Swift and Ceph Rados Gateway, \ - please disable Swift or set REMOTE_CEPH_RGW=False" - else - configure_ceph_remote_radosgw + if [ "$ENABLE_CEPH_RGW" = "True" ]; then + echo_summary "Configuring Rados Gateway with Keystone for Swift" + configure_ceph_embedded_rgw fi fi fi