From c2cfa2df3875ed0653d5825be53eff158eb70d76 Mon Sep 17 00:00:00 2001 From: Ramana Raja Date: Wed, 16 Dec 2015 21:40:38 +0530 Subject: [PATCH] Add Manila support for Ubuntu Change-Id: I74314bfcc6b52d524bb84f2232a988f275b9afbf Co-Authored-By: John Spray --- README.md | 7 +-- devstack/lib/ceph | 88 ++++++++++++++++++++++++++++++++++++-- devstack/override-defaults | 1 + devstack/plugin.sh | 8 ++++ devstack/settings | 19 +++++++- manila/post_test_hook.sh | 86 +++++++++++++++++++++++++++++++++++++ manila/pre_test_hook.sh | 47 ++++++++++++++++++++ 7 files changed, 248 insertions(+), 8 deletions(-) create mode 100644 manila/post_test_hook.sh create mode 100644 manila/pre_test_hook.sh diff --git a/README.md b/README.md index 4ba3df3a..71a2e541 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ As part of ```stack.sh```: * Installs Ceph (client and server) packages * Creates a Ceph cluster for use with openstack services -* Configures Ceph as the storage backend for Cinder, Cinder Backup, Nova & Glance services +* Configures Ceph as the storage backend for Cinder, Cinder Backup, Nova, + Manila, and Glance services * Supports Ceph cluster running local or remote to openstack services As part of ```unstack.sh``` | ```clean.sh```: @@ -32,8 +33,8 @@ This plugin also gets used to configure Ceph as the storage backend for the upst ENABLE_CEPH_$SERVICE=False ``` - _where $SERVICE can be CINDER, C_BAK, GLANCE, or NOVA corresponding to - Cinder, Cinder Backup, Glance, and Nova services respectively._ + _where $SERVICE can be CINDER, C_BAK, GLANCE, MANILA or NOVA corresponding to + Cinder, Cinder Backup, Glance, Manila and Nova services respectively._ * Then run ```stack.sh``` and wait for the _magic_ to happen :) diff --git a/devstack/lib/ceph b/devstack/lib/ceph index 4f474150..91edd54c 100644 --- a/devstack/lib/ceph +++ b/devstack/lib/ceph @@ -67,6 +67,13 @@ CINDER_CEPH_POOL_PGP=${CINDER_CEPH_POOL_PGP:-8} CINDER_CEPH_USER=${CINDER_CEPH_USER:-cinder} CINDER_CEPH_UUID=${CINDER_CEPH_UUID:-$(uuidgen)} +# Manila +CEPHFS_POOL_PG=${CEPHFS_POOL_PG:-8} +CEPHFS_METADATA_POOL=${CEPHFS_CEPH_POOL:-cephfs_metadata} +CEPHFS_DATA_POOL=${CEPHFS_CEPH_POOL:-cephfs_data} +MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila} +MDS_ID=${MDS_ID:-a} + # Set ``CEPH_REPLICAS`` to configure how many replicas are to be # configured for your Ceph cluster. By default we are configuring # only one replica since this is way less CPU and memory intensive. If @@ -187,6 +194,9 @@ if is_ceph_enabled_for_service nova; then sudo ceph osd pool delete $NOVA_CEPH_POOL $NOVA_CEPH_POOL \ --yes-i-really-really-mean-it > /dev/null 2>&1 fi +if is_ceph_enabled_for_service manila; then + sudo ceph auth del client.$MANILA_CEPH_USER +fi } function cleanup_ceph_embedded { @@ -244,6 +254,10 @@ sudo ceph-mon -c ${CEPH_CONF_FILE} --mkfs -i $(hostname) \ --keyring /var/lib/ceph/tmp/keyring.mon.$(hostname) if is_ubuntu; then + # TODO (rraja): Do a Ceph version check. If version >= Infernalis, then + # make sure that user "ceph" is the owner of files within + # ${CEPH_DATA_DIR}. + sudo chown -R ceph ${CEPH_DATA_DIR} sudo touch /var/lib/ceph/mon/ceph-$(hostname)/upstart sudo initctl emit ceph-mon id=$(hostname) else @@ -314,6 +328,12 @@ for rep in ${CEPH_REPLICAS_SEQ}; do fi done +# create a MDS +sudo mkdir -p ${CEPH_DATA_DIR}/mds/ceph-${MDS_ID} +sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create mds.${MDS_ID} \ + mon 'allow profile mds ' osd 'allow rw' mds 'allow' | \ + sudo tee ${CEPH_DATA_DIR}/mds/ceph-${MDS_ID}/keyring + # bootstrap rados gateway sudo mkdir ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname) sudo ceph auth get-or-create client.radosgw.$(hostname) \ @@ -329,6 +349,11 @@ else sudo touch \ ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/{sysvinit,done} fi + +# TODO (rraja): Do a Ceph version check. If version >= Infernalis, then +# make sure that user "ceph" is the owner of files within +# ${CEPH_DATA_DIR}. +sudo chown -R ceph ${CEPH_DATA_DIR} } function configure_ceph_embedded_rgw { @@ -425,6 +450,39 @@ iniset $GLANCE_API_CONF glance_store rbd_store_user $GLANCE_CEPH_USER iniset $GLANCE_API_CONF glance_store rbd_store_pool $GLANCE_CEPH_POOL } +function configure_ceph_manila { + sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CEPHFS_METADATA_POOL} \ + ${CEPHFS_POOL_PG} + sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CEPHFS_DATA_POOL} \ + ${CEPHFS_POOL_PG} + sudo ceph -c ${CEPH_CONF_FILE} fs new cephfs ${CEPHFS_METADATA_POOL} \ + ${CEPHFS_DATA_POOL} + sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create \ + client.${MANILA_CEPH_USER} \ + mon "allow *" \ + osd "allow rw" \ + mds "allow *" | \ + sudo tee ${CEPH_CONF_DIR}/ceph.client.${MANILA_CEPH_USER}.keyring + sudo chown ${STACK_USER}:$(id -g -n $whoami) \ + ${CEPH_CONF_DIR}/ceph.client.${MANILA_CEPH_USER}.keyring + # Enable snapshots in CephFS. + sudo ceph -c ${CEPH_CONF_FILE} mds set allow_new_snaps true \ + --yes-i-really-mean-it +} + +function configure_ceph_embedded_manila { + sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} size \ + ${CEPH_REPLICAS} + sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_METADATA_POOL} size \ + ${CEPH_REPLICAS} + if [[ $CEPH_REPLICAS -ne 1 ]]; then + sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \ + crush_ruleset ${RULE_ID} + sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_METADATA_POOL} \ + crush_ruleset ${RULE_ID} + fi +} + function configure_ceph_embedded_nova { # configure Nova service options, ceph pool, ceph user and ceph key sudo ceph -c ${CEPH_CONF_FILE} osd pool \ @@ -509,13 +567,29 @@ install_package ceph-common function install_ceph { if is_ubuntu; then - wget -q -O- 'https://download.ceph.com/keys/release.asc' \ - | sudo apt-key add - + # use wip-manila development repo until Ceph patches needed by Manila's + # Ceph driver are available in a release package. + if is_ceph_enabled_for_service manila; then + wget -q -O- 'https://download.ceph.com/keys/autobuild.asc' \ + | sudo apt-key add - - echo deb http://ceph.com/debian-${CEPH_RELEASE} $(lsb_release -sc) main \ - | sudo tee /etc/apt/sources.list.d/ceph.list + echo deb \ + http://gitbuilder.ceph.com/ceph-deb-$(lsb_release -sc)-x86_64-basic/ref/wip-manila \ + $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list + else + wget -q -O- 'https://download.ceph.com/keys/release.asc' \ + | sudo apt-key add - + echo deb http://ceph.com/debian-${CEPH_RELEASE} $(lsb_release -sc) \ + main | sudo tee /etc/apt/sources.list.d/ceph.list + + fi + + # Update package repo and restore global variable setting after use. + local tmp_retry_update=$RETRY_UPDATE + RETRY_UPDATE=True install_package ceph radosgw libnss3-tools + RETRY_UPDATE=$tmp_retry_update else # Install directly from distro repos. See LP bug 1521073 for more details. # If distro doesn't carry latest ceph, users can install latest ceph repo @@ -528,10 +602,15 @@ fi # start_ceph() - Start running processes, including screen function start_ceph { if is_ubuntu; then + # TODO (rraja): Do a Ceph version check. If version >= Infernalis, then + # make sure that user "ceph" is the owner of files within + # ${CEPH_DATA_DIR}. + sudo chown -R ceph ${CEPH_DATA_DIR} sudo initctl emit ceph-mon id=$(hostname) for id in $(sudo ceph -c ${CEPH_CONF_FILE} osd ls); do sudo start ceph-osd id=${id} done + sudo start ceph-mds id=${MDS_ID} else sudo service ceph start fi @@ -545,6 +624,7 @@ function stop_ceph { if is_ubuntu; then sudo service ceph-mon-all stop > /dev/null 2>&1 sudo service ceph-osd-all stop > /dev/null 2>&1 + sudo service ceph-mds-all stop > /dev/null 2>&1 else sudo service ceph stop > /dev/null 2>&1 fi diff --git a/devstack/override-defaults b/devstack/override-defaults index 71b98260..2959fa8f 100644 --- a/devstack/override-defaults +++ b/devstack/override-defaults @@ -3,6 +3,7 @@ ENABLE_CEPH_CINDER=$(trueorfalse True ENABLE_CEPH_CINDER) ENABLE_CEPH_C_BAK=$(trueorfalse True ENABLE_CEPH_C_BAK) ENABLE_CEPH_GLANCE=$(trueorfalse True ENABLE_CEPH_GLANCE) +ENABLE_CEPH_MANILA=$(trueorfalse True ENABLE_CEPH_MANILA) ENABLE_CEPH_NOVA=$(trueorfalse True ENABLE_CEPH_NOVA) if [[ $ENABLE_CEPH_CINDER == "True" ]]; then diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 7d5e5024..7aaf7dcc 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -40,6 +40,10 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then echo_summary "Configuring libvirt secret" import_libvirt_secret_ceph fi + if is_ceph_enabled_for_service manila; then + echo_summary "Configuring Manila for Ceph" + configure_ceph_manila + fi if [ "$REMOTE_CEPH" = "False" ]; then if is_ceph_enabled_for_service glance; then @@ -54,6 +58,10 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then echo_summary "Configuring Cinder for Ceph" configure_ceph_embedded_cinder fi + if is_ceph_enabled_for_service manila; then + echo_summary "Configuring Manila for Ceph" + configure_ceph_embedded_manila + fi # FIXME: Fix this once radosgw service is running #echo_summary "Configuring Rados Gateway with Keystone for Swift" diff --git a/devstack/settings b/devstack/settings index 64c86c04..5503de1a 100644 --- a/devstack/settings +++ b/devstack/settings @@ -15,5 +15,22 @@ TEMPEST_STORAGE_PROTOCOL=ceph CEPH_LOOPBACK_DISK_SIZE=8G # Source plugin's lib/ceph - source $CEPH_PLUGIN_DIR/lib/ceph + +# Set Manila related global variables used by Manila's DevStack plugin. +if is_ceph_enabled_for_service manila; then + ENABLED_SHARE_PROTOCOLS="CEPHFS" + MANILA_DEFAULT_SHARE_TYPE=cephfstype + # CephFSNative Driver does not yet support creation of shares from + # snapshot. + MANILA_DEFAULT_SHARE_TYPE_EXTRA_SPECS='snapshot_support=False' + + MANILA_ENABLED_BACKENDS=cephfsnative1 + MANILA_CONFIGURE_GROUPS=cephfsnative1 + + MANILA_OPTGROUP_cephfsnative1_share_driver=manila.share.drivers.cephfs.CephFSNativeDriver + MANILA_OPTGROUP_cephfsnative1_driver_handles_share_servers=False + MANILA_OPTGROUP_cephfsnative1_share_backend_name=CEPHFSNATIVE1 + MANILA_OPTGROUP_cephfsnative1_cephfs_conf_path=${CEPH_CONF_FILE} + MANILA_OPTGROUP_cephfsnative1_cephfs_auth_id=${MANILA_CEPH_USER} +fi diff --git a/manila/post_test_hook.sh b/manila/post_test_hook.sh new file mode 100644 index 00000000..fb000f76 --- /dev/null +++ b/manila/post_test_hook.sh @@ -0,0 +1,86 @@ +#!/bin/bash -xe +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script is executed inside post_test_hook function in devstack gate. + +sudo chown -R jenkins:stack $BASE/new/tempest +sudo chown -R jenkins:stack $BASE/data/tempest +sudo chmod -R o+rx $BASE/new/devstack/files + +# Import devstack functions 'iniset' +source $BASE/new/devstack/functions + +export BACKEND_NAME="CEPHFSNATIVE1" +iniset $BASE/new/tempest/etc/tempest.conf share enable_protocols cephfs +iniset $BASE/new/tempest/etc/tempest.conf share storage_protocol CEPHFS +# Disable tempest config option that enables creation of 'ip' type access +# rules by default during tempest test runs. +iniset $BASE/new/tempest/etc/tempest.conf share enable_ip_rules_for_protocols +iniset $BASE/new/tempest/etc/tempest.conf share capability_snapshot_support False + +iniset $BASE/new/tempest/etc/tempest.conf share backend_names $BACKEND_NAME + +# Set two retries for CI jobs +iniset $BASE/new/tempest/etc/tempest.conf share share_creation_retry_number 2 + +# Suppress errors in cleanup of resources +SUPPRESS_ERRORS=${SUPPRESS_ERRORS_IN_CLEANUP:-True} +iniset $BASE/new/tempest/etc/tempest.conf share suppress_errors_in_cleanup $SUPPRESS_ERRORS + + +# Disable multi_backend tests +RUN_MANILA_MULTI_BACKEND_TESTS=${RUN_MANILA_MULTI_BACKEND_TESTS:-False} +iniset $BASE/new/tempest/etc/tempest.conf share multi_backend $RUN_MANILA_MULTI_BACKEND_TESTS + +# Disable manage/unmanage tests +# CephFSNative driver does not yet support manage and unmanage operations of shares. +RUN_MANILA_MANAGE_TESTS=${RUN_MANILA_MANAGE_TESTS:-False} +iniset $BASE/new/tempest/etc/tempest.conf share run_manage_unmanage_tests $RUN_MANILA_MANAGE_TESTS + +# Enable extend tests +RUN_MANILA_EXTEND_TESTS=${RUN_MANILA_EXTEND_TESTS:-True} +iniset $BASE/new/tempest/etc/tempest.conf share run_extend_tests $RUN_MANILA_EXTEND_TESTS + +# Enable shrink tests +RUN_MANILA_SHRINK_TESTS=${RUN_MANILA_SHRINK_TESTS:-True} +iniset $BASE/new/tempest/etc/tempest.conf share run_shrink_tests $RUN_MANILA_SHRINK_TESTS + +# Disable multi_tenancy tests +iniset $BASE/new/tempest/etc/tempest.conf share multitenancy_enabled False + +# Disable snapshot tests +RUN_MANILA_SNAPSHOT_TESTS=${RUN_MANILA_SNAPSHOT_TESTS:-False} +iniset $BASE/new/tempest/etc/tempest.conf share run_snapshot_tests $RUN_MANILA_SNAPSHOT_TESTS + +# Enable consistency group tests +RUN_MANILA_CG_TESTS=${RUN_MANILA_CG_TESTS:-True} +iniset $BASE/new/tempest/etc/tempest.conf share run_consistency_group_tests $RUN_MANILA_CG_TESTS + +# let us control if we die or not +set +o errexit +cd $BASE/new/tempest + +export MANILA_TEMPEST_CONCURRENCY=${MANILA_TEMPEST_CONCURRENCY:-12} +export MANILA_TESTS=${MANILA_TESTS:-'manila_tempest_tests.tests.api'} + +if [[ "$JOB_NAME" =~ "scenario" ]]; then + echo "Set test set to scenario only" + MANILA_TESTS='manila_tempest_tests.tests.scenario' +fi + +# check if tempest plugin was installed correctly +echo 'import pkg_resources; print list(pkg_resources.iter_entry_points("tempest.test_plugins"))' | python + +echo "Running tempest manila test suites" +sudo -H -u jenkins tox -eall-plugin $MANILA_TESTS -- --concurrency=$MANILA_TEMPEST_CONCURRENCY diff --git a/manila/pre_test_hook.sh b/manila/pre_test_hook.sh new file mode 100644 index 00000000..ff2b4e26 --- /dev/null +++ b/manila/pre_test_hook.sh @@ -0,0 +1,47 @@ +#!/bin/bash -xe +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script is executed inside pre_test_hook function in devstack gate. + +localrc_path=$BASE/new/devstack/localrc +echo "DEVSTACK_GATE_TEMPEST_ALLOW_TENANT_ISOLATION=1" >> $localrc_path +echo "API_RATE_LIMIT=False" >> $localrc_path +echo "TEMPEST_SERVICES+=,manila" >> $localrc_path + +echo "MANILA_USE_DOWNGRADE_MIGRATIONS=True" >> $localrc_path + +# JOB_NAME is defined in openstack-infra/config project +# used by CI/CD, where this script is intended to be used. +if [[ "$JOB_NAME" =~ "multibackend" ]]; then + echo "MANILA_MULTI_BACKEND=True" >> $localrc_path +else + echo "MANILA_MULTI_BACKEND=False" >> $localrc_path +fi + +# Enabling isolated metadata in Neutron is required because +# Tempest creates isolated networks and created vm's in scenario tests don't +# have access to Nova Metadata service. This leads to unavailability of +# created vm's in scenario tests. +echo 'ENABLE_ISOLATED_METADATA=True' >> $localrc_path + +# Go to Tempest dir and checkout stable commit to avoid possible +# incompatibilities for plugin stored in Manila repo. +cd $BASE/new/tempest +source $BASE/new/manila/contrib/ci/common.sh +# In lack of $MANILA_TEMPEST_COMMIT fall back to the old hardcoded +# Tempest commit. +git checkout ${MANILA_TEMPEST_COMMIT:-3b1bb9be3265f} + +# Print current Tempest status +git status