From e810d021082dd4ee08c17203870bbb607a56f724 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 23 Dec 2015 04:43:47 +0000 Subject: [PATCH] Add radosgw (aka rgw) support This patch enables the plugin to setup ceph radosgw and configure it as a swift-compatible endpoint with keystone integration. ENABLE_CEPH_RGW is the new variable introduced, which is False by default, and can be set to True in localrc if radosgw needs to be setup. Fixed couple of other (related) things .... 1) Created rgw specific functions 2) Checking for radosgw & swift co-existence, erroring out early enough 3) Using `osd pool default size = 1` in ceph.conf, which set the replica size to 1 for all pools as the default NOTE: Tested on ubuntu trusty only TODOs ... 1) Not tested on Fedora (F22), radosgw startup has some issues there. Co-Authored-By: Deepak C Shetty Change-Id: I21db4168eb69d107599f6b6ab1668b02b764b2c6 --- README.md | 2 + devstack/lib/ceph | 83 ++++++++++++++++++++------------------ devstack/override-defaults | 4 ++ devstack/plugin.sh | 15 ++----- 4 files changed, 53 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 4ba3df3a..258eb023 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ As part of ```stack.sh```: * Installs Ceph (client and server) packages * Creates a Ceph cluster for use with openstack services * Configures Ceph as the storage backend for Cinder, Cinder Backup, Nova & Glance services +* (Optionally) Sets up & configures Rados gateway (aka rgw or radosgw) as a Swift endpoint with Keystone integration + * Set ```ENABLE_CEPH_RGW=True``` in your ```localrc``` * Supports Ceph cluster running local or remote to openstack services As part of ```unstack.sh``` | ```clean.sh```: diff --git a/devstack/lib/ceph b/devstack/lib/ceph index 4f474150..e640773a 100644 --- a/devstack/lib/ceph +++ b/devstack/lib/ceph @@ -237,6 +237,7 @@ auth_client_required = cephx filestore_xattr_use_omap = true osd crush chooseleaf type = 0 osd journal size = 100 +osd pool default size = ${CEPH_REPLICAS} EOF # bootstrap the ceph monitor @@ -267,14 +268,6 @@ done # so depending on the version we apply different commands local ceph_version ceph_version=$(get_ceph_version) -# change pool replica size according to the CEPH_REPLICAS set by the user -if [[ ${ceph_version%%.*} -eq 0 ]] && [[ ${ceph_version##*.} -lt 87 ]]; then - sudo ceph -c ${CEPH_CONF_FILE} osd pool set rbd size ${CEPH_REPLICAS} - sudo ceph -c ${CEPH_CONF_FILE} osd pool set data size ${CEPH_REPLICAS} - sudo ceph -c ${CEPH_CONF_FILE} osd pool set metadata size ${CEPH_REPLICAS} -else - sudo ceph -c ${CEPH_CONF_FILE} osd pool set rbd size ${CEPH_REPLICAS} -fi # create a simple rule to take OSDs instead of host with CRUSH # then apply this rules to the default pool @@ -314,8 +307,21 @@ for rep in ${CEPH_REPLICAS_SEQ}; do fi done +# Check if radosgw (aka rgw) needs to be setup +if [ "$ENABLE_CEPH_RGW" = "True" ]; then + if is_service_enabled swift; then + die $LINENO \ + "You can not activate both Swift and Ceph Rados Gateway, \ + please disable Swift or set ENABLE_CEPH_RGW=False" + else + configure_ceph_rgw + fi +fi +} + +function configure_ceph_rgw { # bootstrap rados gateway -sudo mkdir ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname) +sudo mkdir -p ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname) sudo ceph auth get-or-create client.radosgw.$(hostname) \ osd 'allow rwx' mon 'allow rw' \ -o /etc/ceph/ceph.client.radosgw.$(hostname).keyring @@ -324,11 +330,30 @@ sudo cp /etc/ceph/ceph.client.radosgw.$(hostname).keyring \ ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/keyring if is_ubuntu; then - sudo touch ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}/{upstart,done} + sudo touch \ + ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/{upstart,done} else sudo touch \ ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/{sysvinit,done} fi + +if [[ ! "$(egrep "\[client.radosgw\]" ${CEPH_CONF_FILE})" ]]; then + cat </dev/null +[client.radosgw.$(hostname)] +host = $(hostname) +keyring = /var/lib/ceph/radosgw/ceph-radosgw.$(hostname)/keyring +rgw socket path = /tmp/radosgw-$(hostname).sock +log file = /var/log/ceph/radosgw-$(hostname).log +rgw data = /var/lib/ceph/radosgw/ceph-radosgw.$(hostname) +rgw print continue = false +rgw frontends = civetweb port=${CEPH_RGW_PORT} +rgw keystone url = http://${SERVICE_HOST}:35357 +rgw keystone admin token = ${SERVICE_TOKEN} +rgw keystone accepted roles = Member, _member_, admin +rgw s3 auth use keystone = true +nss db path = ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss +EOF +fi } function configure_ceph_embedded_rgw { @@ -345,23 +370,8 @@ if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:${CEPH_RGW_PORT}/swift/v1" fi -if [[ ! "$(egrep "\[client.radosgw\]" ${CEPH_CONF_FILE})" ]]; then - cat </dev/null -[client.radosgw.$(hostname)] -host = $(hostname) -keyring = /var/lib/ceph/radosgw/ceph-radosgw.$(hostname)/keyring -rgw socket path = /tmp/radosgw-$(hostname).sock -log file = /var/log/ceph/radosgw-$(hostname).log -rgw data = /var/lib/ceph/radosgw/ceph-radosgw.$(hostname) -rgw print continue = false -rgw frontends = civetweb port=${CEPH_RGW_PORT} -rgw keystone url = http://${SERVICE_HOST}:35357 -rgw keystone admin token = ${SERVICE_TOKEN} -rgw keystone accepted roles = Member, _member_, admin -rgw s3 auth use keystone = true -nss db path = ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss -EOF -fi +# Let keystone generate the certs, rgw needs these. +keystone-manage pki_setup --rebuild # radosgw needs to access keystone's revocation list sudo mkdir ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss @@ -374,9 +384,11 @@ sudo openssl x509 -in /etc/keystone/ssl/certs/signing_cert.pem -pubkey | \ sudo certutil -A \ -d ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/nss \ -n signing_cert -t "P,P,P" + +sudo start radosgw id=radosgw.$(hostname) } -function configure_ceph_remote_radosgw { +function configure_ceph_remote_rgw { if [[ -z "$CEPH_REMOTE_RGW_URL" ]]; then die $LINENO \ "You activated REMOTE_CEPH_RGW thus CEPH_REMOTE_RGW_URL must be defined" @@ -394,8 +406,6 @@ fi function configure_ceph_embedded_glance { # configure Glance service options, ceph pool, ceph user and ceph key -sudo ceph -c ${CEPH_CONF_FILE} osd pool \ -set ${GLANCE_CEPH_POOL} size ${CEPH_REPLICAS} if [[ $CEPH_REPLICAS -ne 1 ]]; then sudo ceph -c ${CEPH_CONF_FILE} osd pool \ set ${GLANCE_CEPH_POOL} crush_ruleset ${RULE_ID} @@ -427,8 +437,6 @@ iniset $GLANCE_API_CONF glance_store rbd_store_pool $GLANCE_CEPH_POOL function configure_ceph_embedded_nova { # configure Nova service options, ceph pool, ceph user and ceph key -sudo ceph -c ${CEPH_CONF_FILE} osd pool \ -set ${NOVA_CEPH_POOL} size ${CEPH_REPLICAS} if [[ $CEPH_REPLICAS -ne 1 ]]; then sudo ceph -c ${CEPH_CONF_FILE} osd pool \ @@ -467,8 +475,6 @@ if ! is_ceph_enabled_for_service cinder; then function configure_ceph_embedded_cinder { # Configure Cinder service options, ceph pool, ceph user and ceph key -sudo ceph -c ${CEPH_CONF_FILE} osd pool \ - set ${CINDER_CEPH_POOL} size ${CEPH_REPLICAS} if [[ $CEPH_REPLICAS -ne 1 ]]; then sudo ceph -c ${CEPH_CONF_FILE} osd pool \ @@ -532,19 +538,18 @@ if is_ubuntu; then for id in $(sudo ceph -c ${CEPH_CONF_FILE} osd ls); do sudo start ceph-osd id=${id} done +# sudo start radosgw id=radosgw.$(hostname) else sudo service ceph start fi - -# FIXME: Some issues with radosgw start, disabling it for now -#sudo service radosgw start } # stop_ceph() - Stop running processes (non-screen) function stop_ceph { if is_ubuntu; then - sudo service ceph-mon-all stop > /dev/null 2>&1 - sudo service ceph-osd-all stop > /dev/null 2>&1 + sudo stop ceph-mon-all > /dev/null 2>&1 + sudo stop ceph-osd-all > /dev/null 2>&1 + sudo stop radosgw-all > /dev/null 2>&1 else sudo service ceph stop > /dev/null 2>&1 fi diff --git a/devstack/override-defaults b/devstack/override-defaults index 71b98260..4168b188 100644 --- a/devstack/override-defaults +++ b/devstack/override-defaults @@ -5,6 +5,10 @@ ENABLE_CEPH_C_BAK=$(trueorfalse True ENABLE_CEPH_C_BAK) ENABLE_CEPH_GLANCE=$(trueorfalse True ENABLE_CEPH_GLANCE) ENABLE_CEPH_NOVA=$(trueorfalse True ENABLE_CEPH_NOVA) +# RGW is not enabled by default as its not used in upstream CI, yet! +# Local devstack setup can enable it in localrc, if needed +ENABLE_CEPH_RGW=$(trueorfalse False ENABLE_CEPH_RGW) + if [[ $ENABLE_CEPH_CINDER == "True" ]]; then CINDER_DRIVER=ceph CINDER_ENABLED_BACKENDS=ceph diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 7d5e5024..64d68428 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -54,18 +54,9 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then echo_summary "Configuring Cinder for Ceph" configure_ceph_embedded_cinder fi - # FIXME: Fix this once radosgw service is running - - #echo_summary "Configuring Rados Gateway with Keystone for Swift" - #configure_ceph_embedded_rgw - fi - if [ "$REMOTE_CEPH_RGW" = "True" ]; then - if is_service_enabled swift; then - die $LINENO \ - "You can not activate both Swift and Ceph Rados Gateway, \ - please disable Swift or set REMOTE_CEPH_RGW=False" - else - configure_ceph_remote_radosgw + if [ "$ENABLE_CEPH_RGW" = "True" ]; then + echo_summary "Configuring Rados Gateway with Keystone for Swift" + configure_ceph_embedded_rgw fi fi fi