Add radosgw (aka rgw) support

This patch enables the plugin to setup ceph radosgw and configure it as a swift-compatible endpoint with keystone integration. ENABLE_CEPH_RGW is the new variable introduced, which is False by default, and can be set to True in localrc if radosgw needs to be setup. Fixed couple of other (related) things .... 1) Created rgw specific functions 2) Checking for radosgw & swift co-existence, erroring out early enough 3) Using `osd pool default size = 1` in ceph.conf, which set the replica size to 1 for all pools as the default NOTE: Tested on ubuntu trusty only TODOs ... 1) Not tested on Fedora (F22), radosgw startup has some issues there. Co-Authored-By: Deepak C Shetty <deepakcs@redhat.com> Change-Id: I21db4168eb69d107599f6b6ab1668b02b764b2c6
2016-01-12 13:22:29 +00:00 · 2016-01-12 13:22:29 +00:00 · 30944248cf
parent c2622a8272
commit 30944248cf
4 changed files with 49 additions and 51 deletions
--- a/README.md
+++ b/README.md
@ -11,6 +11,8 @@ As part of ```stack.sh```:
 * Creates a Ceph cluster for use with openstack services
 * Configures Ceph as the storage backend for Cinder, Cinder Backup, Nova,
  Manila (not by default), and Glance services
+* (Optionally) Sets up & configures Rados gateway (aka rgw or radosgw) as a Swift endpoint with Keystone integration
+  * Set ```ENABLE_CEPH_RGW=True``` in your ```localrc```
 * Supports Ceph cluster running local or remote to openstack services

 As part of ```unstack.sh``` | ```clean.sh```:
--- a/devstack/lib/ceph
+++ b/devstack/lib/ceph
@ -264,6 +264,7 @@ auth_client_required = cephx
 filestore_xattr_use_omap = true
 osd crush chooseleaf type = 0
 osd journal size = 100
+osd pool default size = ${CEPH_REPLICAS}
 EOF

 # bootstrap the ceph monitor
@ -301,14 +302,6 @@ done
 # so depending on the version we apply different commands
 local ceph_version
 ceph_version=$(get_ceph_version mon)
-# change pool replica size according to the CEPH_REPLICAS set by the user
-if [[ ${ceph_version%%.*} -eq 0 ]] && [[ ${ceph_version##*.} -lt 87 ]]; then
-    sudo ceph -c ${CEPH_CONF_FILE} osd pool set rbd size ${CEPH_REPLICAS}
-    sudo ceph -c ${CEPH_CONF_FILE} osd pool set data size ${CEPH_REPLICAS}
-    sudo ceph -c ${CEPH_CONF_FILE} osd pool set metadata size ${CEPH_REPLICAS}
-else
-    sudo ceph -c ${CEPH_CONF_FILE} osd pool set rbd size ${CEPH_REPLICAS}
-fi

 # create a simple rule to take OSDs instead of host with CRUSH
 # then apply this rules to the default pool
@ -361,8 +354,12 @@ if is_ceph_enabled_for_service manila; then
    fi
 fi

+configure_ceph_rgw
+}
+
+function configure_ceph_rgw {
 # bootstrap rados gateway
-sudo mkdir ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)
+sudo mkdir -p ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)
 sudo ceph auth get-or-create client.radosgw.$(hostname) \
 osd 'allow rwx' mon 'allow rw' \
 -o /etc/ceph/ceph.client.radosgw.$(hostname).keyring
@ -371,7 +368,8 @@ sudo cp /etc/ceph/ceph.client.radosgw.$(hostname).keyring \
 ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/keyring

 if is_ubuntu; then
-    sudo touch ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}/{upstart,done}
+    sudo touch \
+    ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/{upstart,done}
 else
    sudo touch \
    ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/{sysvinit,done}
@ -382,6 +380,24 @@ fi
 if [[ $(echo $(get_ceph_version mon) '>=' 9.2 | bc -l) == 1 ]]; then
    sudo chown -R ceph. ${CEPH_DATA_DIR}
 fi
+
+if [[ ! "$(egrep "\[client.radosgw\]" ${CEPH_CONF_FILE})" ]]; then
+    cat <<EOF | sudo tee -a ${CEPH_CONF_FILE}>/dev/null
+[client.radosgw.$(hostname)]
+host = $(hostname)
+keyring = /var/lib/ceph/radosgw/ceph-radosgw.$(hostname)/keyring
+rgw socket path = /tmp/radosgw-$(hostname).sock
+log file = /var/log/ceph/radosgw-$(hostname).log
+rgw data = /var/lib/ceph/radosgw/ceph-radosgw.$(hostname)
+rgw print continue = false
+rgw frontends = civetweb port=${CEPH_RGW_PORT}
+rgw keystone url = http://${SERVICE_HOST}:35357
+rgw keystone admin token = ${SERVICE_TOKEN}
+rgw keystone accepted roles = Member, _member_, admin
+rgw s3 auth use keystone = true
+nss db path = ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss
+EOF
+fi
 }

 function configure_ceph_embedded_rgw {
@ -398,23 +414,8 @@ if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then
    "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:${CEPH_RGW_PORT}/swift/v1"
 fi

-if [[ ! "$(egrep "\[client.radosgw\]" ${CEPH_CONF_FILE})" ]]; then
-    cat <<EOF | sudo tee ${CEPH_CONF_FILE}>/dev/null
-[client.radosgw.$(hostname)]
-host = $(hostname)
-keyring = /var/lib/ceph/radosgw/ceph-radosgw.$(hostname)/keyring
-rgw socket path = /tmp/radosgw-$(hostname).sock
-log file = /var/log/ceph/radosgw-$(hostname).log
-rgw data = /var/lib/ceph/radosgw/ceph-radosgw.$(hostname)
-rgw print continue = false
-rgw frontends = civetweb port=${CEPH_RGW_PORT}
-rgw keystone url = http://${SERVICE_HOST}:35357
-rgw keystone admin token = ${SERVICE_TOKEN}
-rgw keystone accepted roles = Member, _member_, admin
-rgw s3 auth use keystone = true
-nss db path = ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss
-EOF
-fi
+# Let keystone generate the certs, rgw needs these.
+keystone-manage pki_setup --rebuild

 # radosgw needs to access keystone's revocation list
 sudo mkdir ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss
@ -427,9 +428,11 @@ sudo openssl x509 -in /etc/keystone/ssl/certs/signing_cert.pem -pubkey | \
 sudo certutil -A \
 -d ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss \
 -n signing_cert -t "P,P,P"
+
+sudo start radosgw id=radosgw.$(hostname)
 }

-function configure_ceph_remote_radosgw {
+function configure_ceph_remote_rgw {
 if [[ -z "$CEPH_REMOTE_RGW_URL" ]]; then
    die $LINENO \
    "You activated REMOTE_CEPH_RGW thus CEPH_REMOTE_RGW_URL must be defined"
@ -447,8 +450,6 @@ fi

 function configure_ceph_embedded_glance {
 # configure Glance service options, ceph pool, ceph user and ceph key
-sudo ceph -c ${CEPH_CONF_FILE} osd pool \
-set ${GLANCE_CEPH_POOL} size ${CEPH_REPLICAS}
 if [[ $CEPH_REPLICAS -ne 1 ]]; then
    sudo ceph -c ${CEPH_CONF_FILE} osd pool \
    set ${GLANCE_CEPH_POOL} crush_ruleset ${RULE_ID}
@ -511,8 +512,6 @@ function configure_ceph_embedded_manila {

 function configure_ceph_embedded_nova {
 # configure Nova service options, ceph pool, ceph user and ceph key
-sudo ceph -c ${CEPH_CONF_FILE} osd pool \
-set ${NOVA_CEPH_POOL} size ${CEPH_REPLICAS}

 if [[ $CEPH_REPLICAS -ne 1 ]]; then
    sudo ceph -c ${CEPH_CONF_FILE} osd pool \
@ -551,8 +550,6 @@ if ! is_ceph_enabled_for_service cinder; then

 function configure_ceph_embedded_cinder {
 # Configure Cinder service options, ceph pool, ceph user and ceph key
-sudo ceph -c ${CEPH_CONF_FILE} osd pool \
-    set ${CINDER_CEPH_POOL} size ${CEPH_REPLICAS}

 if [[ $CEPH_REPLICAS -ne 1 ]]; then
    sudo ceph -c ${CEPH_CONF_FILE} osd pool \
@ -644,16 +641,14 @@ if is_ubuntu; then
 else
    sudo service ceph start
 fi
-
-# FIXME: Some issues with radosgw start, disabling it for now
-#sudo service radosgw start
 }

 # stop_ceph() - Stop running processes (non-screen)
 function stop_ceph {
 if is_ubuntu; then
-    sudo service ceph-mon-all stop > /dev/null 2>&1
-    sudo service ceph-osd-all stop > /dev/null 2>&1
+    sudo stop ceph-mon-all > /dev/null 2>&1
+    sudo stop ceph-osd-all > /dev/null 2>&1
+    sudo stop radosgw-all > /dev/null 2>&1
    if is_ceph_enabled_for_service manila; then
        sudo service ceph-mds-all stop > /dev/null 2>&1
    fi
--- a/devstack/override-defaults
+++ b/devstack/override-defaults
@ -7,6 +7,9 @@ ENABLE_CEPH_GLANCE=$(trueorfalse True ENABLE_CEPH_GLANCE)
 # CephFS Manila driver is WIP.
 ENABLE_CEPH_MANILA=$(trueorfalse False ENABLE_CEPH_MANILA)
 ENABLE_CEPH_NOVA=$(trueorfalse True ENABLE_CEPH_NOVA)
+# RGW is not enabled by default as its not used in upstream CI, yet!
+# Local devstack setup can enable it in localrc, if needed
+ENABLE_CEPH_RGW=$(trueorfalse False ENABLE_CEPH_RGW)

 if [[ $ENABLE_CEPH_CINDER == "True" ]]; then
    CINDER_DRIVER=ceph
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@ -4,6 +4,13 @@ if [[ "$1" == "source" ]]; then
    # Initial source
    source $TOP_DIR/lib/ceph
 elif [[ "$1" == "stack" && "$2" == "pre-install" ]]; then
+    if [ "$ENABLE_CEPH_RGW" = "True" ]; then
+        if is_service_enabled swift; then
+            die $LINENO \
+            "You can not activate both Swift and Ceph Rados Gateway, \
+            please disable Swift or set ENABLE_CEPH_RGW=False"
+        fi
+    fi
    echo_summary "Installing Ceph"
    check_os_support_ceph
    if [ "$REMOTE_CEPH" = "False" ]; then
@ -62,18 +69,9 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
            echo_summary "Configuring Manila for Ceph"
            configure_ceph_embedded_manila
        fi
-        # FIXME: Fix this once radosgw service is running
-
-        #echo_summary "Configuring Rados Gateway with Keystone for Swift"
-        #configure_ceph_embedded_rgw
-    fi
-    if [ "$REMOTE_CEPH_RGW" = "True" ]; then
-        if is_service_enabled swift; then
-            die $LINENO \
-            "You can not activate both Swift and Ceph Rados Gateway, \
-            please disable Swift or set REMOTE_CEPH_RGW=False"
-        else
-            configure_ceph_remote_radosgw
+        if [ "$ENABLE_CEPH_RGW" = "True" ]; then
+            echo_summary "Configuring Rados Gateway with Keystone for Swift"
+            configure_ceph_embedded_rgw
        fi
    fi
 fi