Add Manila support for Ubuntu

Change-Id: I74314bfcc6b52d524bb84f2232a988f275b9afbf
Co-Authored-By: John Spray <john.spray@redhat.com>

README.md

@@ -9,7 +9,8 @@ As part of ```stack.sh```:
 
 * Installs Ceph (client and server) packages
 * Creates a Ceph cluster for use with openstack services
-* Configures Ceph as the storage backend for Cinder, Cinder Backup, Nova & Glance services
+* Configures Ceph as the storage backend for Cinder, Cinder Backup, Nova,
+  Manila, and Glance services
 * Supports Ceph cluster running local or remote to openstack services
 
 As part of ```unstack.sh``` | ```clean.sh```:
@@ -32,8 +33,8 @@ This plugin also gets used to configure Ceph as the storage backend for the upst
     ENABLE_CEPH_$SERVICE=False
     ```
 
-  _where $SERVICE can be CINDER, C_BAK, GLANCE, or NOVA corresponding to
-  Cinder, Cinder Backup, Glance, and Nova services respectively._
+  _where $SERVICE can be CINDER, C_BAK, GLANCE, MANILA or NOVA corresponding to
+  Cinder, Cinder Backup, Glance, Manila and Nova services respectively._
 
 * Then run ```stack.sh``` and wait for the _magic_ to happen :)
 

devstack/lib/ceph

@@ -67,6 +67,13 @@ CINDER_CEPH_POOL_PGP=${CINDER_CEPH_POOL_PGP:-8}
 CINDER_CEPH_USER=${CINDER_CEPH_USER:-cinder}
 CINDER_CEPH_UUID=${CINDER_CEPH_UUID:-$(uuidgen)}
 
+# Manila
+CEPHFS_POOL_PG=${CEPHFS_POOL_PG:-8}
+CEPHFS_METADATA_POOL=${CEPHFS_CEPH_POOL:-cephfs_metadata}
+CEPHFS_DATA_POOL=${CEPHFS_CEPH_POOL:-cephfs_data}
+MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila}
+MDS_ID=${MDS_ID:-a}
+
 # Set ``CEPH_REPLICAS`` to configure how many replicas are to be
 # configured for your Ceph cluster. By default we are configuring
 # only one replica since this is way less CPU and memory intensive. If
@@ -187,6 +194,9 @@ if is_ceph_enabled_for_service nova; then
     sudo ceph osd pool delete $NOVA_CEPH_POOL $NOVA_CEPH_POOL \
     --yes-i-really-really-mean-it > /dev/null 2>&1
 fi
+if is_ceph_enabled_for_service manila; then
+    sudo ceph auth del client.$MANILA_CEPH_USER
+fi
 }
 
 function cleanup_ceph_embedded {
@@ -244,6 +254,10 @@ sudo ceph-mon -c ${CEPH_CONF_FILE} --mkfs -i $(hostname) \
 --keyring /var/lib/ceph/tmp/keyring.mon.$(hostname)
 
 if is_ubuntu; then
+    # TODO (rraja): Do a Ceph version check. If version >= Infernalis, then
+    # make sure that user "ceph" is the owner of files within
+    # ${CEPH_DATA_DIR}.
+    sudo chown -R ceph ${CEPH_DATA_DIR}
     sudo touch /var/lib/ceph/mon/ceph-$(hostname)/upstart
     sudo initctl emit ceph-mon id=$(hostname)
 else
@@ -314,6 +328,12 @@ for rep in ${CEPH_REPLICAS_SEQ}; do
     fi
 done
 
+# create a MDS
+sudo mkdir -p ${CEPH_DATA_DIR}/mds/ceph-${MDS_ID}
+sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create mds.${MDS_ID} \
+    mon 'allow profile mds ' osd 'allow rw' mds 'allow' | \
+    sudo tee ${CEPH_DATA_DIR}/mds/ceph-${MDS_ID}/keyring
+
 # bootstrap rados gateway
 sudo mkdir ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)
 sudo ceph auth get-or-create client.radosgw.$(hostname) \
@@ -329,6 +349,11 @@ else
     sudo touch \
     ${CEPH_DATA_DIR}/radosgw/ceph-radosgw.$(hostname)/{sysvinit,done}
 fi
+
+# TODO (rraja): Do a Ceph version check. If version >= Infernalis, then
+# make sure that user "ceph" is the owner of files within
+# ${CEPH_DATA_DIR}.
+sudo chown -R ceph ${CEPH_DATA_DIR}
 }
 
 function configure_ceph_embedded_rgw {
@@ -425,6 +450,39 @@ iniset $GLANCE_API_CONF glance_store rbd_store_user $GLANCE_CEPH_USER
 iniset $GLANCE_API_CONF glance_store rbd_store_pool $GLANCE_CEPH_POOL
 }
 
+function configure_ceph_manila {
+    sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CEPHFS_METADATA_POOL} \
+        ${CEPHFS_POOL_PG}
+    sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CEPHFS_DATA_POOL} \
+        ${CEPHFS_POOL_PG}
+    sudo ceph -c ${CEPH_CONF_FILE} fs new cephfs ${CEPHFS_METADATA_POOL} \
+        ${CEPHFS_DATA_POOL}
+    sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create \
+        client.${MANILA_CEPH_USER} \
+        mon "allow *" \
+        osd "allow rw" \
+        mds "allow *" | \
+        sudo tee ${CEPH_CONF_DIR}/ceph.client.${MANILA_CEPH_USER}.keyring
+    sudo chown ${STACK_USER}:$(id -g -n $whoami) \
+        ${CEPH_CONF_DIR}/ceph.client.${MANILA_CEPH_USER}.keyring
+    # Enable snapshots in CephFS.
+    sudo ceph -c ${CEPH_CONF_FILE} mds set allow_new_snaps true \
+        --yes-i-really-mean-it
+}
+
+function configure_ceph_embedded_manila {
+    sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} size \
+        ${CEPH_REPLICAS}
+    sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_METADATA_POOL} size \
+        ${CEPH_REPLICAS}
+    if [[ $CEPH_REPLICAS -ne 1 ]]; then
+        sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \
+            crush_ruleset ${RULE_ID}
+        sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_METADATA_POOL} \
+            crush_ruleset ${RULE_ID}
+    fi
+}
+
 function configure_ceph_embedded_nova {
 # configure Nova service options, ceph pool, ceph user and ceph key
 sudo ceph -c ${CEPH_CONF_FILE} osd pool \
@@ -509,13 +567,29 @@ install_package ceph-common
 
 function install_ceph {
 if is_ubuntu; then
-    wget -q -O- 'https://download.ceph.com/keys/release.asc' \
-    | sudo apt-key add -
+    # use wip-manila development repo until Ceph patches needed by Manila's
+    # Ceph driver are available in a release package.
+    if is_ceph_enabled_for_service manila; then
+        wget -q -O- 'https://download.ceph.com/keys/autobuild.asc' \
+        | sudo apt-key add -
 
-    echo deb http://ceph.com/debian-${CEPH_RELEASE} $(lsb_release -sc) main \
-    | sudo tee /etc/apt/sources.list.d/ceph.list
+        echo deb \
+        http://gitbuilder.ceph.com/ceph-deb-$(lsb_release -sc)-x86_64-basic/ref/wip-manila \
+        $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
+    else
+        wget -q -O- 'https://download.ceph.com/keys/release.asc' \
+        | sudo apt-key add -
 
+        echo deb http://ceph.com/debian-${CEPH_RELEASE} $(lsb_release -sc) \
+        main | sudo tee /etc/apt/sources.list.d/ceph.list
+
+    fi
+
+    # Update package repo and restore global variable setting after use.
+    local tmp_retry_update=$RETRY_UPDATE
+    RETRY_UPDATE=True
     install_package ceph radosgw libnss3-tools
+    RETRY_UPDATE=$tmp_retry_update
 else
     # Install directly from distro repos. See LP bug 1521073 for more details.
     # If distro doesn't carry latest ceph, users can install latest ceph repo
@@ -528,10 +602,15 @@ fi
 # start_ceph() - Start running processes, including screen
 function start_ceph {
 if is_ubuntu; then
+    # TODO (rraja): Do a Ceph version check. If version >= Infernalis, then
+    # make sure that user "ceph" is the owner of files within
+    # ${CEPH_DATA_DIR}.
+    sudo chown -R ceph ${CEPH_DATA_DIR}
     sudo initctl emit ceph-mon id=$(hostname)
     for id in $(sudo ceph -c ${CEPH_CONF_FILE} osd ls); do
         sudo start ceph-osd id=${id}
     done
+    sudo start ceph-mds id=${MDS_ID}
 else
     sudo service ceph start
 fi
@@ -545,6 +624,7 @@ function stop_ceph {
 if is_ubuntu; then
     sudo service ceph-mon-all stop > /dev/null 2>&1
     sudo service ceph-osd-all stop > /dev/null 2>&1
+    sudo service ceph-mds-all stop > /dev/null 2>&1
 else
     sudo service ceph stop > /dev/null 2>&1
 fi

devstack/override-defaults

@@ -3,6 +3,7 @@
 ENABLE_CEPH_CINDER=$(trueorfalse True ENABLE_CEPH_CINDER)
 ENABLE_CEPH_C_BAK=$(trueorfalse True ENABLE_CEPH_C_BAK)
 ENABLE_CEPH_GLANCE=$(trueorfalse True ENABLE_CEPH_GLANCE)
+ENABLE_CEPH_MANILA=$(trueorfalse True ENABLE_CEPH_MANILA)
 ENABLE_CEPH_NOVA=$(trueorfalse True ENABLE_CEPH_NOVA)
 
 if [[ $ENABLE_CEPH_CINDER == "True" ]]; then

devstack/plugin.sh

@@ -40,6 +40,10 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
         echo_summary "Configuring libvirt secret"
         import_libvirt_secret_ceph
     fi
+    if is_ceph_enabled_for_service manila; then
+        echo_summary "Configuring Manila for Ceph"
+        configure_ceph_manila
+    fi
 
     if [ "$REMOTE_CEPH" = "False" ]; then
         if is_ceph_enabled_for_service glance; then
@@ -54,6 +58,10 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
             echo_summary "Configuring Cinder for Ceph"
             configure_ceph_embedded_cinder
         fi
+        if is_ceph_enabled_for_service manila; then
+            echo_summary "Configuring Manila for Ceph"
+            configure_ceph_embedded_manila
+        fi
         # FIXME: Fix this once radosgw service is running
 
         #echo_summary "Configuring Rados Gateway with Keystone for Swift"

devstack/settings

@@ -15,5 +15,22 @@ TEMPEST_STORAGE_PROTOCOL=ceph
 CEPH_LOOPBACK_DISK_SIZE=8G
 
 # Source plugin's lib/ceph
-
 source $CEPH_PLUGIN_DIR/lib/ceph
+
+# Set Manila related global variables used by Manila's DevStack plugin.
+if is_ceph_enabled_for_service manila; then
+    ENABLED_SHARE_PROTOCOLS="CEPHFS"
+    MANILA_DEFAULT_SHARE_TYPE=cephfstype
+    # CephFSNative Driver does not yet support creation of shares from
+    # snapshot.
+    MANILA_DEFAULT_SHARE_TYPE_EXTRA_SPECS='snapshot_support=False'
+
+    MANILA_ENABLED_BACKENDS=cephfsnative1
+    MANILA_CONFIGURE_GROUPS=cephfsnative1
+
+    MANILA_OPTGROUP_cephfsnative1_share_driver=manila.share.drivers.cephfs.CephFSNativeDriver
+    MANILA_OPTGROUP_cephfsnative1_driver_handles_share_servers=False
+    MANILA_OPTGROUP_cephfsnative1_share_backend_name=CEPHFSNATIVE1
+    MANILA_OPTGROUP_cephfsnative1_cephfs_conf_path=${CEPH_CONF_FILE}
+    MANILA_OPTGROUP_cephfsnative1_cephfs_auth_id=${MANILA_CEPH_USER}
+fi

manila/post_test_hook.sh

@@ -0,0 +1,86 @@
+#!/bin/bash -xe
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This script is executed inside post_test_hook function in devstack gate.
+
+sudo chown -R jenkins:stack $BASE/new/tempest
+sudo chown -R jenkins:stack $BASE/data/tempest
+sudo chmod -R o+rx $BASE/new/devstack/files
+
+# Import devstack functions 'iniset'
+source $BASE/new/devstack/functions
+
+export BACKEND_NAME="CEPHFSNATIVE1"
+iniset $BASE/new/tempest/etc/tempest.conf share enable_protocols cephfs
+iniset $BASE/new/tempest/etc/tempest.conf share storage_protocol CEPHFS
+# Disable tempest config option that enables creation of 'ip' type access
+# rules by default during tempest test runs.
+iniset $BASE/new/tempest/etc/tempest.conf share enable_ip_rules_for_protocols
+iniset $BASE/new/tempest/etc/tempest.conf share capability_snapshot_support False
+
+iniset $BASE/new/tempest/etc/tempest.conf share backend_names $BACKEND_NAME
+
+# Set two retries for CI jobs
+iniset $BASE/new/tempest/etc/tempest.conf share share_creation_retry_number 2
+
+# Suppress errors in cleanup of resources
+SUPPRESS_ERRORS=${SUPPRESS_ERRORS_IN_CLEANUP:-True}
+iniset $BASE/new/tempest/etc/tempest.conf share suppress_errors_in_cleanup $SUPPRESS_ERRORS
+
+
+# Disable multi_backend tests
+RUN_MANILA_MULTI_BACKEND_TESTS=${RUN_MANILA_MULTI_BACKEND_TESTS:-False}
+iniset $BASE/new/tempest/etc/tempest.conf share multi_backend $RUN_MANILA_MULTI_BACKEND_TESTS
+
+# Disable manage/unmanage tests
+# CephFSNative driver does not yet support manage and unmanage operations of shares.
+RUN_MANILA_MANAGE_TESTS=${RUN_MANILA_MANAGE_TESTS:-False}
+iniset $BASE/new/tempest/etc/tempest.conf share run_manage_unmanage_tests $RUN_MANILA_MANAGE_TESTS
+
+# Enable extend tests
+RUN_MANILA_EXTEND_TESTS=${RUN_MANILA_EXTEND_TESTS:-True}
+iniset $BASE/new/tempest/etc/tempest.conf share run_extend_tests $RUN_MANILA_EXTEND_TESTS
+
+# Enable shrink tests
+RUN_MANILA_SHRINK_TESTS=${RUN_MANILA_SHRINK_TESTS:-True}
+iniset $BASE/new/tempest/etc/tempest.conf share run_shrink_tests $RUN_MANILA_SHRINK_TESTS
+
+# Disable multi_tenancy tests
+iniset $BASE/new/tempest/etc/tempest.conf share multitenancy_enabled False
+
+# Disable snapshot tests
+RUN_MANILA_SNAPSHOT_TESTS=${RUN_MANILA_SNAPSHOT_TESTS:-False}
+iniset $BASE/new/tempest/etc/tempest.conf share run_snapshot_tests $RUN_MANILA_SNAPSHOT_TESTS
+
+# Enable consistency group tests
+RUN_MANILA_CG_TESTS=${RUN_MANILA_CG_TESTS:-True}
+iniset $BASE/new/tempest/etc/tempest.conf share run_consistency_group_tests $RUN_MANILA_CG_TESTS
+
+# let us control if we die or not
+set +o errexit
+cd $BASE/new/tempest
+
+export MANILA_TEMPEST_CONCURRENCY=${MANILA_TEMPEST_CONCURRENCY:-12}
+export MANILA_TESTS=${MANILA_TESTS:-'manila_tempest_tests.tests.api'}
+
+if [[ "$JOB_NAME" =~ "scenario" ]]; then
+    echo "Set test set to scenario only"
+    MANILA_TESTS='manila_tempest_tests.tests.scenario'
+fi
+
+# check if tempest plugin was installed correctly
+echo 'import pkg_resources; print list(pkg_resources.iter_entry_points("tempest.test_plugins"))' | python
+
+echo "Running tempest manila test suites"
+sudo -H -u jenkins tox -eall-plugin $MANILA_TESTS -- --concurrency=$MANILA_TEMPEST_CONCURRENCY

manila/pre_test_hook.sh

@@ -0,0 +1,47 @@
+#!/bin/bash -xe
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This script is executed inside pre_test_hook function in devstack gate.
+
+localrc_path=$BASE/new/devstack/localrc
+echo "DEVSTACK_GATE_TEMPEST_ALLOW_TENANT_ISOLATION=1" >> $localrc_path
+echo "API_RATE_LIMIT=False" >> $localrc_path
+echo "TEMPEST_SERVICES+=,manila" >> $localrc_path
+
+echo "MANILA_USE_DOWNGRADE_MIGRATIONS=True" >> $localrc_path
+
+# JOB_NAME is defined in openstack-infra/config project
+# used by CI/CD, where this script is intended to be used.
+if [[ "$JOB_NAME" =~ "multibackend" ]]; then
+    echo "MANILA_MULTI_BACKEND=True" >> $localrc_path
+else
+    echo "MANILA_MULTI_BACKEND=False" >> $localrc_path
+fi
+
+# Enabling isolated metadata in Neutron is required because
+# Tempest creates isolated networks and created vm's in scenario tests don't
+# have access to Nova Metadata service. This leads to unavailability of
+# created vm's in scenario tests.
+echo 'ENABLE_ISOLATED_METADATA=True' >> $localrc_path
+
+# Go to Tempest dir and checkout stable commit to avoid possible
+# incompatibilities for plugin stored in Manila repo.
+cd $BASE/new/tempest
+source $BASE/new/manila/contrib/ci/common.sh
+# In lack of $MANILA_TEMPEST_COMMIT fall back to the old hardcoded
+# Tempest commit.
+git checkout ${MANILA_TEMPEST_COMMIT:-3b1bb9be3265f}
+
+# Print current Tempest status
+git status