diff --git a/.zuul.yaml b/.zuul.yaml index a43635fd..7c0cd925 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -90,10 +90,11 @@ parent: manila-tempest-plugin-cephfs-native-cephadm - job: - name: devstack-plugin-ceph-cephfs-nfs + name: devstack-plugin-ceph-cephfs-nfs-standalone description: | Runs manila tempest plugin tests with CephFS via NFS-Ganesha as a manila - back end (DHSS=False) + back end (DHSS=False). The Ceph cluster is created with cephadm + while nfs-ganesha is installed "standalone" via a package. parent: manila-tempest-plugin-cephfs-nfs - job: @@ -174,7 +175,7 @@ - devstack-plugin-ceph-cephfs-native: irrelevant-files: *irrelevant-files voting: false - - devstack-plugin-ceph-cephfs-nfs: + - devstack-plugin-ceph-cephfs-nfs-standalone: irrelevant-files: *irrelevant-files voting: false # - devstack-plugin-ceph-master-tempest: diff --git a/devstack/lib/ceph b/devstack/lib/ceph index 7939a876..98b78f19 100755 --- a/devstack/lib/ceph +++ b/devstack/lib/ceph @@ -21,6 +21,7 @@ # Save trace setting XTRACE=$(set +o | grep xtrace) set +o xtrace +source $CEPH_PLUGIN_DIR/lib/common # Defaults @@ -30,20 +31,6 @@ TEST_MASTER=$(trueorfalse False TEST_MASTER) CEPH_RELEASE=${CEPH_RELEASE:-quincy} -GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'} -# Remove "v" and "-stable" prefix/suffix tags -GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//") - -if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then - # default ganesha release based on ceph release - case $CEPH_RELEASE in - pacific) - GANESHA_RELEASE='3.5' ;; - *) - GANESHA_RELEASE='4.0' ;; - esac -fi - # Deploy a Ceph demo container instead of a non-containerized version CEPH_CONTAINERIZED=$(trueorfalse False CEPH_CONTAINERIZED) @@ -111,10 +98,6 @@ CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data} MANILA_CEPH_DRIVER=${MANILA_CEPH_DRIVER:-cephfsnative} MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila} -# Allows driver to store NFS-Ganesha exports and export counter as -# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later, -# Ceph v12.2.2 or later, and OpenStack Queens or later. -MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True} # Set ``CEPH_REPLICAS`` to configure how many replicas are to be # configured for your Ceph cluster. By default we are configuring @@ -757,67 +740,13 @@ function configure_ceph_manila { if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then configure_nfs_ganesha - # NFS-Ganesha server cannot run alongwith with other kernel NFS server. - sudo systemctl stop nfs-server || true - sudo systemctl disable nfs-server || true - sudo systemctl enable nfs-ganesha - sudo systemctl start nfs-ganesha || ( - echo "Ganesha didn't start. Let's debug..." >&2 - sudo systemctl status nfs-ganesha || true - echo "**Ganesha conf file**" >&2 - sudo cat /etc/ganesha/ganesha.conf || true - echo "**Ganesha log file**" >&2 - sudo cat /var/log/ganesha/ganesha.log || true - echo "**Exiting**" >&2 - exit 1 - ) - echo "Ganesha started successfully!" >&2 + start_nfs_ganesha fi # RESTART DOCKER CONTAINER } -function configure_nfs_ganesha { - # Configure NFS-Ganesha to work with Manila's CephFS driver - sudo mkdir -p /etc/ganesha/export.d - if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then - # Create an empty placeholder ganesha export index object - echo | sudo rados -p ${CEPHFS_DATA_POOL} put ganesha-export-index - - cat </dev/null -RADOS_URLS { - ceph_conf = ${CEPH_CONF_FILE}; - userid = admin; -} - -CACHEINODE { - Dir_Max = 1; - Dir_Chunk = 0; - - Cache_FDs = false; - - NParts = 1; - Cache_Size = 1; -} - -EXPORT_DEFAULTS { - Attr_Expiration_Time = 0; -} - -%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index -EOF - else - sudo touch /etc/ganesha/export.d/INDEX.conf - echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf - fi -} - -function cleanup_nfs_ganesha { - sudo systemctl stop nfs-ganesha - sudo systemctl disable nfs-ganesha - sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs -} - function configure_ceph_embedded_manila { if [[ $CEPH_REPLICAS -ne 1 ]]; then sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \ @@ -958,19 +887,6 @@ EOF sudo dnf config-manager --add-repo ceph.repo } -function dnf_add_repository_nfsganesha { - local repo="" - - case $ganesha_release in - 3.*) - repo="centos-release-nfs-ganesha30" ;; - *) - repo="centos-release-nfs-ganesha4" ;; - esac - - sudo dnf -y install ${repo} -} - # configure_repo_ceph() - Configure Ceph repositories # Usage: configure_repo_ceph # - package_release: to override the os_RELEASE variable @@ -1009,44 +925,11 @@ function cleanup_repo_ceph { fi } -# configure_repo_nfsganesha() - Configure NFS Ganesha repositories -function configure_repo_nfsganesha { - if is_ubuntu; then - # NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major - # version; we can't use a build microversion unlike el8/el9 builds - if [[ $GANESHA_RELEASE =~ 3 ]]; then - sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0 - sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0 - elif [[ $GANESHA_RELEASE =~ 4 ]]; then - sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-4 - sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-4 - else - die $LINENO "NFS-Ganesha $GANESHA_RELEASE is not supported by the Ceph plugin for Devstack" - fi - sudo apt-get -y update - elif is_fedora; then - dnf_add_repository_nfsganesha - fi -} - -# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories -# Usage: cleanup_repo_nfsganesha -function cleanup_repo_nfsganesha { - if is_ubuntu; then - sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*" - elif is_fedora; then - sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo - fi -} - function setup_packages_for_manila_on_ubuntu { CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2" if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then - configure_repo_nfsganesha - LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}" - CEPH_PACKAGES="${CEPH_PACKAGES} $LIBNTIRPC_PACKAGE \ - nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-urls nfs-ganesha-vfs" + install_nfs_ganesha fi if python3_enabled; then @@ -1056,9 +939,7 @@ function setup_packages_for_manila_on_ubuntu { function setup_packages_for_manila_on_fedora_family { if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then - configure_repo_nfsganesha - CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph \ - nfs-ganesha-rados-urls nfs-ganesha-vfs" + install_nfs_ganesha fi } @@ -1175,8 +1056,7 @@ function stop_ceph { fi if is_ceph_enabled_for_service manila; then if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then - sudo systemctl stop nfs-ganesha - sudo systemctl disable nfs-ganesha + stop_nfs_ganesha fi sudo systemctl stop ceph-mds@${MDS_ID} sudo systemctl disable ceph-mds@${MDS_ID} diff --git a/devstack/lib/cephadm b/devstack/lib/cephadm index 8b36fccd..85f823dd 100755 --- a/devstack/lib/cephadm +++ b/devstack/lib/cephadm @@ -67,7 +67,11 @@ MDS_LOGS=${MDS_LOGS:-"False"} # MANILA DEFAULTS MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila} -# NFS OPTIONS +# NFS OPTIONS: Only apply when ENABLE_CEPH_MANILA=True +# Whether or not cephadm should deploy/manage NFS-Ganesha? If set to False, +# we'll deploy a "standalone" NFS Ganesha instead, not managed by cephadm. +CEPHADM_DEPLOY_NFS=${CEPHADM_DEPLOY_NFS:-True} +# Clustered NFS Options FSNAME=${FSNAME:-'cephfs'} NFS_PORT=2049 CEPHFS_CLIENT=0 @@ -153,7 +157,7 @@ function export_spec { # Pre-install ceph: install required dependencies function install_deps { - if [[ "$REMOTE_CEPH" == "False" ]]; then + if [[ "$REMOTE_CEPH" = "False" ]]; then install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests fi } @@ -218,16 +222,16 @@ function start_ceph { --skip-mon-network \ --mon-ip "$HOST_IP" - test -e $CEPH_CONFIG - test -e $CEPH_KEYRING + test -e $CEPH_CONFIG + test -e $CEPH_KEYRING - if [ "$CEPHADM_DEV_OSD" == 'True' ]; then - create_osd_dev - fi - # Wait cephadm backend to be operational - # and add osds via drivegroups - sleep "$SLEEP" - add_osds + if [ "$CEPHADM_DEV_OSD" == 'True' ]; then + create_osd_dev + fi + # Wait cephadm backend to be operational + # and add osds via drivegroups + sleep "$SLEEP" + add_osds fi } @@ -271,7 +275,7 @@ function add_osds { while [ "$ATTEMPTS" -ne 0 ]; do num_osds=$($SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ - --keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds') + --keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds') if [ "$num_osds" -ge "$MIN_OSDS" ]; then break; fi @@ -342,8 +346,8 @@ function create_keys { # Install ceph: add MDS function cephfs_config { # Two pools are generated by this action - # - $FSNAME.FSNAME.data - # - $FSNAME.FSNAME.meta + # - cephfs.$FSNAME.data + # - cephfs.$FSNAME.meta # and the mds daemon is deployed $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph fs volume create "$FSNAME" @@ -359,8 +363,7 @@ function _get_ceph_version { echo $ceph_version_str } -# Install ceph: add NFS -function ceph_nfs_config { +function _install_and_configure_clustered_nfs { local ceph_version ceph_version=$(_get_ceph_version) @@ -379,6 +382,23 @@ function ceph_nfs_config { fi } +function _install_and_configure_standalone_nfs { + source $CEPH_PLUGIN_DIR/lib/common + install_nfs_ganesha + configure_nfs_ganesha + start_nfs_ganesha +} + +# Install ceph: add NFS +function ceph_nfs_config { + if [[ "$CEPHADM_DEPLOY_NFS" == "True" ]]; then + _install_and_configure_clustered_nfs + else + _install_and_configure_standalone_nfs + fi + +} + function _create_swift_endpoint { local swift_service @@ -482,17 +502,17 @@ function configure_ceph_manila { function enable_services { for item in "${SERVICES[@]}"; do case "$item" in - cephfs|CEPHFS) + cephfs|CEPHFS) echo "[CEPHADM] Config cephfs volume on node $HOSTNAME" cephfs_config CEPHFS_CLIENT=1 ;; - nfs|NFS) + nfs|NFS) echo "[CEPHADM] Deploying NFS on node $HOSTNAME" ceph_nfs_config CEPHFS_CLIENT=1 ;; - rgw|RGW) + rgw|RGW) echo "[CEPHADM] Deploying RGW on node $HOSTNAME" rgw ;; @@ -735,8 +755,14 @@ function cleanup_ceph { # purge ceph config file and keys $SUDO rm -f ${CEPH_CONF_DIR}/* if is_ceph_enabled_for_service nova; then - _undefine_virsh_secret + _undefine_virsh_secret fi + if [[ "$CEPHADM_DEPLOY_NFS" != "True" ]]; then + stop_nfs_ganesha + cleanup_nfs_ganesha + cleanup_repo_nfs_ganesha + fi + } function disable_cephadm { diff --git a/devstack/lib/common b/devstack/lib/common new file mode 100755 index 00000000..1b7b665c --- /dev/null +++ b/devstack/lib/common @@ -0,0 +1,149 @@ +#!/bin/bash + +# Allows driver to store NFS-Ganesha exports and export counter as +# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later, +# Ceph v12.2.2 or later, and OpenStack Queens or later. +MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True} +GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'} +# Remove "v" and "-stable" prefix/suffix tags +GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//") +if [[ "$CEPHADM_DEPLOY" = "True" ]]; then + FSNAME=${FSNAME:-'cephfs'} + CEPHFS_DATA_POOL="cephfs.$FSNAME.data" +else + CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data} +fi + +if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then + # default ganesha release based on ceph release + case $CEPH_RELEASE in + pacific) + GANESHA_RELEASE='3.5' + ;; + *) + GANESHA_RELEASE='5.0' + ;; + esac +fi + +# configure_repo_nfsganesha - Configure NFS Ganesha repositories +function configure_repo_nfsganesha { + if is_ubuntu; then + # NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major + # version; we can't use a build microversion unlike el8/el9 builds + case $GANESHA_RELEASE in + 3.*) + sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0 + sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0 + ;; + *) + GANESHA_PPA_VERSION="${GANESHA_RELEASE:0:1}" + sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-"$GANESHA_PPA_VERSION" + sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-"$GANESHA_PPA_VERSION" + ;; + esac + sudo apt-get -y update + elif is_fedora; then + local repo="" + case $GANESHA_RELEASE in + 3.*) + repo="centos-release-nfs-ganesha30" + ;; + *) + repo="centos-release-nfs-ganesha5" + ;; + esac + sudo dnf -y install ${repo} + fi +} + +function install_nfs_ganesha { + configure_repo_nfsganesha + NFS_GANESHA_PACKAGES="nfs-ganesha nfs-ganesha-ceph \ + nfs-ganesha-rados-urls nfs-ganesha-vfs" + if is_ubuntu; then + LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}" + NFS_GANESHA_PACKAGES="${LIBNTIRPC_PACKAGE} ${NFS_GANESHA_PACKAGES}" + fi + install_package $NFS_GANESHA_PACKAGES +} + +function configure_nfs_ganesha { + # Configure NFS-Ganesha to work with Manila's CephFS driver + rados_cmd="sudo rados -p ${CEPHFS_DATA_POOL}" + if [[ "$CEPHADM_DEPLOY" = "True" ]]; then + CEPHADM=${TARGET_BIN}/cephadm + rados_cmd="sudo $CEPHADM shell rados -p ${CEPHFS_DATA_POOL}" + fi + + + sudo mkdir -p /etc/ganesha/export.d + if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then + # Create an empty placeholder ganesha export index object + echo | $rados_cmd put ganesha-export-index - + cat </dev/null + RADOS_URLS { + ceph_conf = ${CEPH_CONF_FILE}; + userid = admin; +} + +CACHEINODE { + Dir_Max = 1; + Dir_Chunk = 0; + + Cache_FDs = false; + + NParts = 1; + Cache_Size = 1; +} + +EXPORT_DEFAULTS { + Attr_Expiration_Time = 0; +} + +%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index +EOF + else + sudo touch /etc/ganesha/export.d/INDEX.conf + echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf + fi +} + +function start_nfs_ganesha { + # NFS-Ganesha server cannot run alongwith with other kernel NFS server. + sudo systemctl stop nfs-server || true + sudo systemctl disable nfs-server || true + sudo systemctl enable nfs-ganesha + sudo systemctl start nfs-ganesha || ( + echo "Ganesha didn't start. Let's debug..." >&2 + sudo systemctl status nfs-ganesha || true + echo "**Ganesha conf file**" >&2 + sudo cat /etc/ganesha/ganesha.conf || true + echo "**Ganesha log file**" >&2 + sudo cat /var/log/ganesha/ganesha.log || true + echo "**Exiting**" >&2 + exit 1 + ) + echo "Standalone NFS-Ganesha started successfully!" >&2 +} + +function stop_nfs_ganesha { + sudo systemctl stop nfs-ganesha + sudo systemctl disable nfs-ganesha +} + +function cleanup_nfs_ganesha { + sudo systemctl stop nfs-ganesha + sudo systemctl disable nfs-ganesha + sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs +} + +# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories +# Usage: cleanup_repo_nfsganesha +function cleanup_repo_nfsganesha { + if is_ubuntu; then + sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*" + elif is_fedora; then + sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo + fi +} diff --git a/devstack/settings b/devstack/settings index 01f74a72..ef9e4dbc 100644 --- a/devstack/settings +++ b/devstack/settings @@ -62,16 +62,16 @@ if (is_ceph_enabled_for_service manila); then MANILA_OPTGROUP_cephfsnfs1_cephfs_conf_path=${CEPH_CONF_FILE} MANILA_OPTGROUP_cephfsnfs1_cephfs_auth_id=${MANILA_CEPH_USER} MANILA_OPTGROUP_cephfsnfs1_cephfs_protocol_helper_type=NFS - MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP - MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE) - if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then - MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE} - MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL} - fi - - if [ "$CEPHADM_DEPLOY" = "True" ]; then + if [[ $CEPHADM_DEPLOY_NFS == "True" ]]; then MANILA_OPTGROUP_cephfsnfs1_cephfs_nfs_cluster_id=${FSNAME} + else + MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP + MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE) + if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then + MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE} + MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL} + fi fi fi fi