Add radosgw (aka rgw) support

This patch enables the plugin to setup ceph radosgw and configures
it as a swift-compatible endpoint with keystone integration.

ENABLE_CEPH_RGW is the new variable introduced, which is False
by default, and can be set to True in localrc if radosgw needs
to be setup.

Fixed couple of other (related) things ...
1) Created rgw specific functions
2) Checking for radosgw & swift co-existence, erroring out early enough

NOTE: Tested on ubuntu trusty only

TODOs ...
1) Not tested on Fedora (F22), radosgw startup has some issues there.

Co-Authored-By: Deepak C Shetty <deepakcs@redhat.com>
Change-Id: I21db4168eb69d107599f6b6ab1668b02b764b2c6

README.md

@@ -11,6 +11,8 @@ As part of ```stack.sh```:
 * Creates a Ceph cluster for use with openstack services
 * Configures Ceph as the storage backend for Cinder, Cinder Backup, Nova,
   Manila (not by default), and Glance services
+* (Optionally) Sets up & configures Rados gateway (aka rgw or radosgw) as a Swift endpoint with Keystone integration
+  * Set ```ENABLE_CEPH_RGW=True``` in your ```localrc```
 * Supports Ceph cluster running local or remote to openstack services
 
 As part of ```unstack.sh``` | ```clean.sh```:
@@ -50,9 +52,17 @@ This plugin also gets used to configure Ceph as the storage backend for the upst
 * Then run ```stack.sh``` and wait for the _magic_ to happen :)
 
 
+# Known Issues / Limitations
+
+* Rados Gateway with Keystone for Swift - works on Ubuntu only
+* Tempest test failures when using RGW as swift endpoint
+* Tempest fails due to verify-tempest-config erroring out, when using RGW as swift endpoint
+  * Patch sent @ https://review.openstack.org/#/c/264179/
+
+
 # TODOs
 
-* Configuring Rados Gateway with Keystone for Swift
+* Fix Rados Gateway with Keystone for Swift on Fedora
 * Add support for Ceph Infernalis release
 * Add support for distro specific ceph repos
 * Add Manila support for non-Ubuntu systems

devstack/lib/ceph

@@ -217,7 +217,10 @@ fi
 }
 
 function cleanup_ceph_embedded {
-sudo killall -w -9 ceph-mon ceph-osd radosgw
+sudo killall -w -9 ceph-mon ceph-osd
+if [ "$ENABLE_CEPH_RGW" = "True" ]; then
+    sudo killall -w -9 radosgw
+fi
 sudo rm -rf ${CEPH_DATA_DIR}/*/*
 if egrep -q ${CEPH_DATA_DIR} /proc/mounts; then
     sudo umount ${CEPH_DATA_DIR}
@@ -361,17 +364,21 @@ if is_ceph_enabled_for_service manila; then
     fi
 fi
 
+if [ "$ENABLE_CEPH_RGW" = "True" ]; then
+    configure_ceph_rgw
+fi
+}
+
+function configure_ceph_rgw {
 # bootstrap rados gateway
-sudo mkdir ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)
+sudo mkdir -p ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)
 sudo ceph auth get-or-create client.radosgw.$(hostname) \
 osd 'allow rwx' mon 'allow rw' \
--o /etc/ceph/ceph.client.radosgw.$(hostname).keyring
+-o ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/keyring
-
-sudo cp /etc/ceph/ceph.client.radosgw.$(hostname).keyring \
-${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/keyring
 
 if is_ubuntu; then
-    sudo touch ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}/{upstart,done}
+    sudo touch \
+    ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/{upstart,done}
 else
     sudo touch \
     ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/{sysvinit,done}
@@ -382,6 +389,24 @@ fi
 if [[ $(echo $(get_ceph_version mon) '>=' 9.2 | bc -l) == 1 ]]; then
     sudo chown -R ceph. ${CEPH_DATA_DIR}
 fi
+
+if [[ ! "$(egrep "\[client.radosgw\]" ${CEPH_CONF_FILE})" ]]; then
+    cat <<EOF | sudo tee -a ${CEPH_CONF_FILE}>/dev/null
+[client.radosgw.$(hostname)]
+host = $(hostname)
+keyring = ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/keyring
+rgw socket path = /tmp/radosgw-$(hostname).sock
+log file = /var/log/ceph/radosgw-$(hostname).log
+rgw data = ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)
+rgw print continue = false
+rgw frontends = civetweb port=${CEPH_RGW_PORT}
+rgw keystone url = http://${SERVICE_HOST}:35357
+rgw keystone admin token = ${SERVICE_TOKEN}
+rgw keystone accepted roles = Member, _member_, admin
+rgw s3 auth use keystone = true
+nss db path = ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss
+EOF
+fi
 }
 
 function configure_ceph_embedded_rgw {
@@ -398,23 +423,8 @@ if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then
     "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:${CEPH_RGW_PORT}/swift/v1"
 fi
 
-if [[ ! "$(egrep "\[client.radosgw\]" ${CEPH_CONF_FILE})" ]]; then
+# Let keystone generate the certs, rgw needs these.
-    cat <<EOF | sudo tee ${CEPH_CONF_FILE}>/dev/null
+keystone-manage pki_setup --rebuild
-[client.radosgw.$(hostname)]
-host = $(hostname)
-keyring = /var/lib/ceph/radosgw/ceph-radosgw.$(hostname)/keyring
-rgw socket path = /tmp/radosgw-$(hostname).sock
-log file = /var/log/ceph/radosgw-$(hostname).log
-rgw data = /var/lib/ceph/radosgw/ceph-radosgw.$(hostname)
-rgw print continue = false
-rgw frontends = civetweb port=${CEPH_RGW_PORT}
-rgw keystone url = http://${SERVICE_HOST}:35357
-rgw keystone admin token = ${SERVICE_TOKEN}
-rgw keystone accepted roles = Member, _member_, admin
-rgw s3 auth use keystone = true
-nss db path = ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss
-EOF
-fi
 
 # radosgw needs to access keystone's revocation list
 sudo mkdir ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss
@@ -427,9 +437,13 @@ sudo openssl x509 -in /etc/keystone/ssl/certs/signing_cert.pem -pubkey | \
 sudo certutil -A \
 -d ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss \
 -n signing_cert -t "P,P,P"
+
+# radosgw service is started here as it needs the keystone pki_setup as a
+# pre-requisite
+sudo start radosgw id=radosgw.$(hostname)
 }
 
-function configure_ceph_remote_radosgw {
+function configure_ceph_remote_rgw {
 if [[ -z "$CEPH_REMOTE_RGW_URL" ]]; then
     die $LINENO \
     "You activated REMOTE_CEPH_RGW thus CEPH_REMOTE_RGW_URL must be defined"
@@ -583,7 +597,9 @@ function init_ceph {
 # make sure to kill all ceph processes first
 sudo pkill -f ceph-mon || true
 sudo pkill -f ceph-osd || true
-sudo pkill -f radosgw || true
+if [ "$ENABLE_CEPH_RGW" = "True" ]; then
+    sudo pkill -f radosgw || true
+fi
 if is_ceph_enabled_for_service manila; then
     sudo pkill -f ceph-mds || true
 fi
@@ -608,7 +624,7 @@ if is_ubuntu; then
 
         # Update package repo.
         REPOS_UPDATED=False
-        install_package ceph ceph-mds radosgw libnss3-tools
+        install_package ceph ceph-mds libnss3-tools
     else
         wget -q -O- 'https://download.ceph.com/keys/release.asc' \
         | sudo apt-key add -
@@ -618,15 +634,21 @@ if is_ubuntu; then
 
         # Update package repo.
         REPOS_UPDATED=False
-        install_package ceph radosgw libnss3-tools
+        install_package ceph libnss3-tools
     fi
 
+    if [ "$ENABLE_CEPH_RGW" = "True" ]; then
+        install_package radosgw
+    fi
 else
     # Install directly from distro repos. See LP bug 1521073 for more details.
     # If distro doesn't carry latest ceph, users can install latest ceph repo
     # for their distro (if available) from download.ceph.com and then do
     # stack.sh
-    install_package ceph ceph-radosgw
+    install_package ceph
+    if [ "$ENABLE_CEPH_RGW" = "True" ]; then
+        install_package ceph-radosgw
+    fi
 fi
 }
 
@@ -648,16 +670,16 @@ if is_ubuntu; then
 else
     sudo service ceph start
 fi
-
-# FIXME: Some issues with radosgw start, disabling it for now
-#sudo service radosgw start
 }
 
 # stop_ceph() - Stop running processes (non-screen)
 function stop_ceph {
 if is_ubuntu; then
-    sudo service ceph-mon-all stop > /dev/null 2>&1
+    sudo stop ceph-mon-all > /dev/null 2>&1
-    sudo service ceph-osd-all stop > /dev/null 2>&1
+    sudo stop ceph-osd-all > /dev/null 2>&1
+    if [ "$ENABLE_CEPH_RGW" = "True" ]; then
+        sudo stop radosgw-all > /dev/null 2>&1
+    fi
     if is_ceph_enabled_for_service manila; then
         sudo service ceph-mds-all stop > /dev/null 2>&1
     fi

devstack/override-defaults

@@ -7,6 +7,8 @@ ENABLE_CEPH_GLANCE=$(trueorfalse True ENABLE_CEPH_GLANCE)
 # CephFS Manila driver is WIP.
 ENABLE_CEPH_MANILA=$(trueorfalse False ENABLE_CEPH_MANILA)
 ENABLE_CEPH_NOVA=$(trueorfalse True ENABLE_CEPH_NOVA)
+# Do not enable RGW by default as RGW is not tested in upstream CI.
+ENABLE_CEPH_RGW=$(trueorfalse False ENABLE_CEPH_RGW)
 
 if [[ $ENABLE_CEPH_CINDER == "True" ]]; then
     CINDER_DRIVER=ceph

devstack/plugin.sh

@@ -4,6 +4,11 @@ if [[ "$1" == "source" ]]; then
     # Initial source
     source $TOP_DIR/lib/ceph
 elif [[ "$1" == "stack" && "$2" == "pre-install" ]]; then
+    if [[ "$ENABLE_CEPH_RGW" = "True" ]] && (is_service_enabled swift); then
+        die $LINENO \
+        "You cannot activate both Swift and Ceph Rados Gateway, \
+        please disable Swift or set ENABLE_CEPH_RGW=False"
+    fi
     echo_summary "Installing Ceph"
     check_os_support_ceph
     if [ "$REMOTE_CEPH" = "False" ]; then
@@ -62,18 +67,9 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
             echo_summary "Configuring Manila for Ceph"
             configure_ceph_embedded_manila
         fi
-        # FIXME: Fix this once radosgw service is running
+        if [ "$ENABLE_CEPH_RGW" = "True" ]; then
-
+            echo_summary "Configuring Rados Gateway with Keystone for Swift"
-        #echo_summary "Configuring Rados Gateway with Keystone for Swift"
+            configure_ceph_embedded_rgw
-        #configure_ceph_embedded_rgw
-    fi
-    if [ "$REMOTE_CEPH_RGW" = "True" ]; then
-        if is_service_enabled swift; then
-            die $LINENO \
-            "You can not activate both Swift and Ceph Rados Gateway, \
-            please disable Swift or set REMOTE_CEPH_RGW=False"
-        else
-            configure_ceph_remote_radosgw
         fi
     fi
 fi