From 11260fcbc95a9b13a91fef5ff259fd998651254c Mon Sep 17 00:00:00 2001 From: Goutham Pacha Ravi Date: Mon, 1 Apr 2024 17:11:05 -0700 Subject: [PATCH] Standalone nfs-ganesha with cephadm deployment Manila supports using a standalone NFS-Ganesha server as well as a ceph orchestrator deployed NFS-Ganesha cluster ("ceph nfs service"). We've only ever allowed using ceph orch deployed NFS with ceph orch deployed clusters through this devstack plugin. With this change, the plugin can optionally deploy a standalone NFS-Ganesha service with a ceph orch deployed ceph cluster. This will greatly simplify testing when we sunset the package based installation/deployment of ceph. Depends-On: I2198eee3892b2bb0eb835ec66e21b708152b33a9 Change-Id: If983bb5d5a5fc0c16c1cead84b5fa30ea961d21b Implements: bp/cephadm-deploy Signed-off-by: Goutham Pacha Ravi (cherry picked from commit ca2486efb408094683848b3f4cd1e551ea266872) (cherry picked from commit af28bdab3919eb2dc714a87ef0625717b5bfa938) (cherry picked from commit 2a7fca87fea7b01cbc1f7161f68293ba99d475ea) --- .zuul.yaml | 7 +- devstack/lib/ceph | 130 ++----------------------------------- devstack/lib/cephadm | 80 +++++++++++++++++------ devstack/lib/common | 149 +++++++++++++++++++++++++++++++++++++++++++ devstack/settings | 16 ++--- 5 files changed, 225 insertions(+), 157 deletions(-) create mode 100755 devstack/lib/common diff --git a/.zuul.yaml b/.zuul.yaml index fe1351e1..acdb2ddf 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -78,10 +78,11 @@ parent: manila-tempest-plugin-cephfs-native-cephadm - job: - name: devstack-plugin-ceph-cephfs-nfs + name: devstack-plugin-ceph-cephfs-nfs-standalone description: | Runs manila tempest plugin tests with CephFS via NFS-Ganesha as a manila - back end (DHSS=False) + back end (DHSS=False). The Ceph cluster is created with cephadm + while nfs-ganesha is installed "standalone" via a package. parent: manila-tempest-plugin-cephfs-nfs - job: @@ -156,7 +157,7 @@ - devstack-plugin-ceph-cephfs-native: irrelevant-files: *irrelevant-files voting: false - - devstack-plugin-ceph-cephfs-nfs: + - devstack-plugin-ceph-cephfs-nfs-standalone: irrelevant-files: *irrelevant-files voting: false # - devstack-plugin-ceph-tempest-fedora-latest diff --git a/devstack/lib/ceph b/devstack/lib/ceph index a054d9ac..7e154279 100755 --- a/devstack/lib/ceph +++ b/devstack/lib/ceph @@ -21,6 +21,7 @@ # Save trace setting XTRACE=$(set +o | grep xtrace) set +o xtrace +source $CEPH_PLUGIN_DIR/lib/common # Defaults @@ -30,20 +31,6 @@ TEST_MASTER=$(trueorfalse False TEST_MASTER) CEPH_RELEASE=${CEPH_RELEASE:-pacific} -GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'} -# Remove "v" and "-stable" prefix/suffix tags -GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//") - -if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then - # default ganesha release based on ceph release - case $CEPH_RELEASE in - pacific) - GANESHA_RELEASE='3.5' ;; - *) - GANESHA_RELEASE='4.0' ;; - esac -fi - # Deploy a Ceph demo container instead of a non-containerized version CEPH_CONTAINERIZED=$(trueorfalse False CEPH_CONTAINERIZED) @@ -111,10 +98,6 @@ CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data} MANILA_CEPH_DRIVER=${MANILA_CEPH_DRIVER:-cephfsnative} MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila} -# Allows driver to store NFS-Ganesha exports and export counter as -# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later, -# Ceph v12.2.2 or later, and OpenStack Queens or later. -MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True} # Set ``CEPH_REPLICAS`` to configure how many replicas are to be # configured for your Ceph cluster. By default we are configuring @@ -757,67 +740,13 @@ function configure_ceph_manila { if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then configure_nfs_ganesha - # NFS-Ganesha server cannot run alongwith with other kernel NFS server. - sudo systemctl stop nfs-server || true - sudo systemctl disable nfs-server || true - sudo systemctl enable nfs-ganesha - sudo systemctl start nfs-ganesha || ( - echo "Ganesha didn't start. Let's debug..." >&2 - sudo systemctl status nfs-ganesha || true - echo "**Ganesha conf file**" >&2 - sudo cat /etc/ganesha/ganesha.conf || true - echo "**Ganesha log file**" >&2 - sudo cat /var/log/ganesha/ganesha.log || true - echo "**Exiting**" >&2 - exit 1 - ) - echo "Ganesha started successfully!" >&2 + start_nfs_ganesha fi # RESTART DOCKER CONTAINER } -function configure_nfs_ganesha { - # Configure NFS-Ganesha to work with Manila's CephFS driver - sudo mkdir -p /etc/ganesha/export.d - if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then - # Create an empty placeholder ganesha export index object - echo | sudo rados -p ${CEPHFS_DATA_POOL} put ganesha-export-index - - cat </dev/null -RADOS_URLS { - ceph_conf = ${CEPH_CONF_FILE}; - userid = admin; -} - -CACHEINODE { - Dir_Max = 1; - Dir_Chunk = 0; - - Cache_FDs = false; - - NParts = 1; - Cache_Size = 1; -} - -EXPORT_DEFAULTS { - Attr_Expiration_Time = 0; -} - -%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index -EOF - else - sudo touch /etc/ganesha/export.d/INDEX.conf - echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf - fi -} - -function cleanup_nfs_ganesha { - sudo systemctl stop nfs-ganesha - sudo systemctl disable nfs-ganesha - sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs -} - function configure_ceph_embedded_manila { if [[ $CEPH_REPLICAS -ne 1 ]]; then sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \ @@ -958,19 +887,6 @@ EOF sudo dnf config-manager --add-repo ceph.repo } -function dnf_add_repository_nfsganesha { - local repo="" - - case $ganesha_release in - 3.*) - repo="centos-release-nfs-ganesha30" ;; - *) - repo="centos-release-nfs-ganesha4" ;; - esac - - sudo dnf -y install ${repo} -} - # configure_repo_ceph() - Configure Ceph repositories # Usage: configure_repo_ceph # - package_release: to override the os_RELEASE variable @@ -1009,44 +925,11 @@ function cleanup_repo_ceph { fi } -# configure_repo_nfsganesha() - Configure NFS Ganesha repositories -function configure_repo_nfsganesha { - if is_ubuntu; then - # NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major - # version; we can't use a build microversion unlike el8/el9 builds - if [[ $GANESHA_RELEASE =~ 3 ]]; then - sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0 - sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0 - elif [[ $GANESHA_RELEASE =~ 4 ]]; then - sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-4 - sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-4 - else - die $LINENO "NFS-Ganesha $GANESHA_RELEASE is not supported by the Ceph plugin for Devstack" - fi - sudo apt-get -y update - elif is_fedora; then - dnf_add_repository_nfsganesha - fi -} - -# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories -# Usage: cleanup_repo_nfsganesha -function cleanup_repo_nfsganesha { - if is_ubuntu; then - sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*" - elif is_fedora; then - sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo - fi -} - function setup_packages_for_manila_on_ubuntu { CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2" if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then - configure_repo_nfsganesha - LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}" - CEPH_PACKAGES="${CEPH_PACKAGES} $LIBNTIRPC_PACKAGE \ - nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-urls nfs-ganesha-vfs" + install_nfs_ganesha fi if python3_enabled; then @@ -1056,9 +939,7 @@ function setup_packages_for_manila_on_ubuntu { function setup_packages_for_manila_on_fedora_family { if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then - configure_repo_nfsganesha - CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph \ - nfs-ganesha-rados-urls nfs-ganesha-vfs" + install_nfs_ganesha fi } @@ -1175,8 +1056,7 @@ function stop_ceph { fi if is_ceph_enabled_for_service manila; then if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then - sudo systemctl stop nfs-ganesha - sudo systemctl disable nfs-ganesha + stop_nfs_ganesha fi sudo systemctl stop ceph-mds@${MDS_ID} sudo systemctl disable ceph-mds@${MDS_ID} diff --git a/devstack/lib/cephadm b/devstack/lib/cephadm index be1b71d9..7f5c5e3b 100755 --- a/devstack/lib/cephadm +++ b/devstack/lib/cephadm @@ -65,7 +65,11 @@ RBD_CLIENT_LOG=/var/log/ceph/qemu-guest-\$pid.log # MANILA DEFAULTS MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila} -# NFS OPTIONS +# NFS OPTIONS: Only apply when ENABLE_CEPH_MANILA=True +# Whether or not cephadm should deploy/manage NFS-Ganesha? If set to False, +# we'll deploy a "standalone" NFS Ganesha instead, not managed by cephadm. +CEPHADM_DEPLOY_NFS=${CEPHADM_DEPLOY_NFS:-True} +# Clustered NFS Options FSNAME=${FSNAME:-'cephfs'} NFS_PORT=2049 CEPHFS_CLIENT=0 @@ -123,7 +127,7 @@ function export_spec { # Pre-install ceph: install required dependencies function install_deps { - if [[ "$REMOTE_CEPH" == "False" ]]; then + if [[ "$REMOTE_CEPH" = "False" ]]; then install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests fi } @@ -182,16 +186,16 @@ function start_ceph { --skip-mon-network \ --mon-ip "$HOST_IP" - test -e $CEPH_CONFIG - test -e $CEPH_KEYRING + test -e $CEPH_CONFIG + test -e $CEPH_KEYRING - if [ "$CEPHADM_DEV_OSD" == 'True' ]; then - create_osd_dev - fi - # Wait cephadm backend to be operational - # and add osds via drivegroups - sleep "$SLEEP" - add_osds + if [ "$CEPHADM_DEV_OSD" == 'True' ]; then + create_osd_dev + fi + # Wait cephadm backend to be operational + # and add osds via drivegroups + sleep "$SLEEP" + add_osds fi } @@ -235,7 +239,7 @@ function add_osds { while [ "$ATTEMPTS" -ne 0 ]; do num_osds=$($SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ - --keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds') + --keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds') if [ "$num_osds" -ge "$MIN_OSDS" ]; then break; fi @@ -306,22 +310,50 @@ function create_keys { # Install ceph: add MDS function cephfs_config { # Two pools are generated by this action - # - $FSNAME.FSNAME.data - # - $FSNAME.FSNAME.meta + # - cephfs.$FSNAME.data + # - cephfs.$FSNAME.meta # and the mds daemon is deployed $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph fs volume create "$FSNAME" } -# Install ceph: add NFS -function ceph_nfs_config { - # (fpantano) TODO: Build an ingress daemon on top of this +# Get Ceph version +function _get_ceph_version { + local ceph_version_str + + ceph_version_str=$(sudo podman run --rm --entrypoint ceph $CONTAINER_IMAGE \ + --version | awk '{ print $3 }') + + echo $ceph_version_str +} + +function _install_and_configure_clustered_nfs { + local ceph_version + ceph_version=$(_get_ceph_version) + echo "[CEPHADM] Deploy nfs.$FSNAME backend" $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph orch apply nfs \ "$FSNAME" --placement="$HOSTNAME" --port $NFS_PORT } +function _install_and_configure_standalone_nfs { + source $CEPH_PLUGIN_DIR/lib/common + install_nfs_ganesha + configure_nfs_ganesha + start_nfs_ganesha +} + +# Install ceph: add NFS +function ceph_nfs_config { + if [[ "$CEPHADM_DEPLOY_NFS" == "True" ]]; then + _install_and_configure_clustered_nfs + else + _install_and_configure_standalone_nfs + fi + +} + function _create_swift_endpoint { local swift_service @@ -425,17 +457,17 @@ function configure_ceph_manila { function enable_services { for item in "${SERVICES[@]}"; do case "$item" in - cephfs|CEPHFS) + cephfs|CEPHFS) echo "[CEPHADM] Config cephfs volume on node $HOSTNAME" cephfs_config CEPHFS_CLIENT=1 ;; - nfs|NFS) + nfs|NFS) echo "[CEPHADM] Deploying NFS on node $HOSTNAME" ceph_nfs_config CEPHFS_CLIENT=1 ;; - rgw|RGW) + rgw|RGW) echo "[CEPHADM] Deploying RGW on node $HOSTNAME" rgw ;; @@ -676,8 +708,14 @@ function cleanup_ceph { # purge ceph config file and keys $SUDO rm -f ${CEPH_CONF_DIR}/* if is_ceph_enabled_for_service nova; then - _undefine_virsh_secret + _undefine_virsh_secret fi + if [[ "$CEPHADM_DEPLOY_NFS" != "True" ]]; then + stop_nfs_ganesha + cleanup_nfs_ganesha + cleanup_repo_nfs_ganesha + fi + } function disable_cephadm { diff --git a/devstack/lib/common b/devstack/lib/common new file mode 100755 index 00000000..1b7b665c --- /dev/null +++ b/devstack/lib/common @@ -0,0 +1,149 @@ +#!/bin/bash + +# Allows driver to store NFS-Ganesha exports and export counter as +# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later, +# Ceph v12.2.2 or later, and OpenStack Queens or later. +MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True} +GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'} +# Remove "v" and "-stable" prefix/suffix tags +GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//") +if [[ "$CEPHADM_DEPLOY" = "True" ]]; then + FSNAME=${FSNAME:-'cephfs'} + CEPHFS_DATA_POOL="cephfs.$FSNAME.data" +else + CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data} +fi + +if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then + # default ganesha release based on ceph release + case $CEPH_RELEASE in + pacific) + GANESHA_RELEASE='3.5' + ;; + *) + GANESHA_RELEASE='5.0' + ;; + esac +fi + +# configure_repo_nfsganesha - Configure NFS Ganesha repositories +function configure_repo_nfsganesha { + if is_ubuntu; then + # NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major + # version; we can't use a build microversion unlike el8/el9 builds + case $GANESHA_RELEASE in + 3.*) + sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0 + sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0 + ;; + *) + GANESHA_PPA_VERSION="${GANESHA_RELEASE:0:1}" + sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-"$GANESHA_PPA_VERSION" + sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-"$GANESHA_PPA_VERSION" + ;; + esac + sudo apt-get -y update + elif is_fedora; then + local repo="" + case $GANESHA_RELEASE in + 3.*) + repo="centos-release-nfs-ganesha30" + ;; + *) + repo="centos-release-nfs-ganesha5" + ;; + esac + sudo dnf -y install ${repo} + fi +} + +function install_nfs_ganesha { + configure_repo_nfsganesha + NFS_GANESHA_PACKAGES="nfs-ganesha nfs-ganesha-ceph \ + nfs-ganesha-rados-urls nfs-ganesha-vfs" + if is_ubuntu; then + LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}" + NFS_GANESHA_PACKAGES="${LIBNTIRPC_PACKAGE} ${NFS_GANESHA_PACKAGES}" + fi + install_package $NFS_GANESHA_PACKAGES +} + +function configure_nfs_ganesha { + # Configure NFS-Ganesha to work with Manila's CephFS driver + rados_cmd="sudo rados -p ${CEPHFS_DATA_POOL}" + if [[ "$CEPHADM_DEPLOY" = "True" ]]; then + CEPHADM=${TARGET_BIN}/cephadm + rados_cmd="sudo $CEPHADM shell rados -p ${CEPHFS_DATA_POOL}" + fi + + + sudo mkdir -p /etc/ganesha/export.d + if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then + # Create an empty placeholder ganesha export index object + echo | $rados_cmd put ganesha-export-index - + cat </dev/null + RADOS_URLS { + ceph_conf = ${CEPH_CONF_FILE}; + userid = admin; +} + +CACHEINODE { + Dir_Max = 1; + Dir_Chunk = 0; + + Cache_FDs = false; + + NParts = 1; + Cache_Size = 1; +} + +EXPORT_DEFAULTS { + Attr_Expiration_Time = 0; +} + +%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index +EOF + else + sudo touch /etc/ganesha/export.d/INDEX.conf + echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf + fi +} + +function start_nfs_ganesha { + # NFS-Ganesha server cannot run alongwith with other kernel NFS server. + sudo systemctl stop nfs-server || true + sudo systemctl disable nfs-server || true + sudo systemctl enable nfs-ganesha + sudo systemctl start nfs-ganesha || ( + echo "Ganesha didn't start. Let's debug..." >&2 + sudo systemctl status nfs-ganesha || true + echo "**Ganesha conf file**" >&2 + sudo cat /etc/ganesha/ganesha.conf || true + echo "**Ganesha log file**" >&2 + sudo cat /var/log/ganesha/ganesha.log || true + echo "**Exiting**" >&2 + exit 1 + ) + echo "Standalone NFS-Ganesha started successfully!" >&2 +} + +function stop_nfs_ganesha { + sudo systemctl stop nfs-ganesha + sudo systemctl disable nfs-ganesha +} + +function cleanup_nfs_ganesha { + sudo systemctl stop nfs-ganesha + sudo systemctl disable nfs-ganesha + sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs +} + +# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories +# Usage: cleanup_repo_nfsganesha +function cleanup_repo_nfsganesha { + if is_ubuntu; then + sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*" + elif is_fedora; then + sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo + fi +} diff --git a/devstack/settings b/devstack/settings index 01f74a72..ef9e4dbc 100644 --- a/devstack/settings +++ b/devstack/settings @@ -62,16 +62,16 @@ if (is_ceph_enabled_for_service manila); then MANILA_OPTGROUP_cephfsnfs1_cephfs_conf_path=${CEPH_CONF_FILE} MANILA_OPTGROUP_cephfsnfs1_cephfs_auth_id=${MANILA_CEPH_USER} MANILA_OPTGROUP_cephfsnfs1_cephfs_protocol_helper_type=NFS - MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP - MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE) - if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then - MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE} - MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL} - fi - - if [ "$CEPHADM_DEPLOY" = "True" ]; then + if [[ $CEPHADM_DEPLOY_NFS == "True" ]]; then MANILA_OPTGROUP_cephfsnfs1_cephfs_nfs_cluster_id=${FSNAME} + else + MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP + MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE) + if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then + MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE} + MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL} + fi fi fi fi