Cleanup installation and revive cephfs-nfs job

The cephfs-nfs job was turned off [1] for perma-failing. This commit adds the original non-voting job back into the check queue and fixes some installation issues: 1) use ceph "quincy" release: Ceph Pacific's end of life is 2023-06-01 [2]. The manila community thinks deployers are more likely to use quincy with the 2023.2 (bobcat) release of OpenStack. 2) run the job with centos-stream-9: There are no packages currently available for Jammy Jellyfish on download.ceph.com [3]. The OS shouldn't really matter for this CI job that is meant to test feature functionality provided by manila. At this time, we'd like to stick with builds provided by the ceph community instead of the distro since it may take a while to get bugfixes into distro builds. 3) The install script uses "nfs-ganesha" builds for ubuntu and centos hosted by the nfs-ganesha community [4]. We will not rely on the ceph community to provide the latest builds for nfs-ganesha any longer. This commit also cleans up the unnecessary condition in the ceph script file pertaining to configuring ceph packages for Jammy Jellyfish. This step wasn't doing anything. Ubuntu packages don't work at the moment and that requires some more investigation. [1] Id2ae61979505de5efb47ce90a2bac8aac2fc5484 [2] https://docs.ceph.com/en/latest/releases/ [3] https://www.spinics.net/lists/ceph-users/msg74312.html [4] https://download.nfs-ganesha.org/ Change-Id: I40dfecfbbe21b2f4b3e4efd903980b5b146c4202 Signed-off-by: Goutham Pacha Ravi <gouthampravi@gmail.com>
2023-03-10 18:44:50 -08:00
parent 863a01b032
commit 563cb5deeb
4 changed files with 82 additions and 118 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -93,6 +93,22 @@
      Runs manila tempest plugin tests with CephFS via NFS-Ganesha as a manila
      back end (DHSS=False)
    parent: manila-tempest-plugin-cephfs-nfs
+    nodeset: devstack-single-node-centos-9-stream
+    vars:
+      # TODO(gouthamr): some tests are disabled due to bugs
+      # IPv6 Tests: https://bugs.launchpad.net/manila/+bug/1998489
+      # snapshot clone fs sync: https://bugs.launchpad.net/manila/+bug/1989273
+      tempest_exclude_regex: "\
+      (^manila_tempest_tests.tests.scenario.*IPv6.*)|\
+      (^manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFS.test_write_data_to_share_created_from_snapshot)"
+      devstack_localrc:
+        MANILA_OPTGROUP_cephfsnfs_cephfs_ganesha_server_ip: "{{ hostvars[inventory_hostname]['nodepool']['private_ipv4'] }}"
+        CEPH_RELEASE: "quincy"
+        MANILA_SETUP_IPV6: false
+        NEUTRON_CREATE_INITIAL_NETWORKS: true
+        IP_VERSION: 4
+
+

 - job:
    name: devstack-plugin-ceph-tempest-fedora-latest
@@ -176,9 +192,9 @@
        - devstack-plugin-ceph-cephfs-native:
            irrelevant-files: *irrelevant-files
            voting: false
-        # - devstack-plugin-ceph-cephfs-nfs:
-        #     irrelevant-files: *irrelevant-files
-        #     voting: false
+        - devstack-plugin-ceph-cephfs-nfs:
+            irrelevant-files: *irrelevant-files
+            voting: false
        # - devstack-plugin-ceph-tempest-fedora-latest
        # - devstack-plugin-ceph-multinode-tempest-py3
        # - devstack-plugin-ceph-multinode-tempest-cephadm:
--- a/devstack/files/debs/devstack-plugin-ceph
+++ b/devstack/files/debs/devstack-plugin-ceph
@@ -0,0 +1 @@
+xfsprogs
--- a/devstack/files/rpms/devstack-plugin-ceph
+++ b/devstack/files/rpms/devstack-plugin-ceph
@@ -0,0 +1,2 @@
+xfsprogs
+dbus-tools
--- a/devstack/lib/ceph
+++ b/devstack/lib/ceph
@@ -30,7 +30,19 @@ TEST_MASTER=$(trueorfalse False TEST_MASTER)

 CEPH_RELEASE=${CEPH_RELEASE:-pacific}

-GANESHA_RELEASE=${GANESHA_RELEASE:-V3.5-stable}
+GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'}
+# Remove "v" and "-stable" prefix/suffix tags
+GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//")
+
+if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then
+    # default ganesha release based on ceph release
+    case $CEPH_RELEASE in
+        pacific)
+            GANESHA_RELEASE='3.5' ;;
+        *)
+            GANESHA_RELEASE='4.0' ;;
+    esac
+fi

 # Deploy a Ceph demo container instead of a non-containerized version
 CEPH_CONTAINERIZED=$(trueorfalse False CEPH_CONTAINERIZED)
@@ -273,17 +285,13 @@ function _undefine_virsh_secret {

 # check_os_support_ceph() - Check if the OS provides a decent version of Ceph
 function check_os_support_ceph {
-    if [[ ! ${DISTRO} =~ (jammy|focal|bionic|xenial|f31|f32|f33|f34|rhel8) ]]; then
-        echo "WARNING: your distro $DISTRO does not provide \
-            (at least) the Luminous release. \
-            Please use Ubuntu Xenial, Ubuntu Bionic, Ubuntu Focal, Ubuntu Jammy, \
-            Fedora 31-34 or CentOS Stream 8."
+    if [[ ! ${DISTRO} =~ (jammy|focal|bionic|xenial|f31|f32|f33|f34|rhel8|rhel9) ]]; then
+        echo "WARNING: devstack-plugin-ceph hasn't been tested with $DISTRO. \
+        Set FORCE_CEPH_INSTALL=yes in your local.conf if you'd like to \
+        attempt installation anyway."
        if [[ "$FORCE_CEPH_INSTALL" != "yes" ]]; then
-            die $LINENO "If you wish to install Ceph on this distribution \
-                anyway run with FORCE_CEPH_INSTALL=yes, \
-                this assumes that YOU will setup the proper repositories"
+            die $LINENO "Not proceeding with install."
        fi
-        NO_UPDATE_REPOS=False
    fi

    if [[ ! $INIT_SYSTEM == 'systemd' ]]; then
@@ -893,7 +901,7 @@ function configure_ceph_cinder {
    sudo rbd pool init ${CINDER_CEPH_POOL}
 }

-# install_ceph() - Collect source and prepare
+# install_ceph_remote() - Collect source and prepare
 function install_ceph_remote {
    install_package ceph-common
    # ceph-common in Bionic (18.04) installs only the python2 variants of
@@ -920,12 +928,12 @@ function install_ceph_remote {

 function dnf_add_repository_ceph {
    local ceph_release=$1
-    local distro_release=$2
+    local package_release=$2

    cat > ceph.repo <<EOF
 [ceph]
 name=Ceph packages for \$basearch
-baseurl=https://download.ceph.com/rpm-${ceph_release}/${distro_release}/\$basearch
+baseurl=https://download.ceph.com/rpm-${ceph_release}/${package_release}/\$basearch
 enabled=1
 priority=2
 gpgcheck=1
@@ -933,7 +941,7 @@ gpgkey=https://download.ceph.com/keys/release.asc

 [ceph-noarch]
 name=Ceph noarch packages
-baseurl=https://download.ceph.com/rpm-${ceph_release}/${distro_release}/noarch
+baseurl=https://download.ceph.com/rpm-${ceph_release}/${package_release}/noarch
 enabled=1
 priority=2
 gpgcheck=1
@@ -941,7 +949,7 @@ gpgkey=https://download.ceph.com/keys/release.asc

 [ceph-source]
 name=Ceph source packages
-baseurl=https://download.ceph.com/rpm-${ceph_release}/${distro_release}/SRPMS
+baseurl=https://download.ceph.com/rpm-${ceph_release}/${package_release}/SRPMS
 enabled=0
 priority=2
 gpgcheck=1
@@ -951,64 +959,42 @@ EOF
 }

 function dnf_add_repository_nfsganesha {
-    local ganesha_release=$1
-    local ceph_release=$2
+    local repo=""

-    cat > nfs-ganesha.repo <<EOF
-[nfs-ganesha]
-name=NFS Ganesha packages for \$basearch
-baseurl=https://download.ceph.com/nfs-ganesha/${ganesha_release}/${ceph_release}/el8/\$basearch
-enabled=1
-priority=2
-gpgcheck=1
-gpgkey=https://download.ceph.com/keys/release.asc
+    case $ganesha_release in
+      3.*)
+        repo="centos-release-nfs-ganesha30" ;;
+      *)
+        repo="centos-release-nfs-ganesha4" ;;
+    esac

-[nfs-ganesha-noarch]
-name=NFS Ganesha noarch packages
-baseurl=https://download.ceph.com/nfs-ganesha/${ganesha_release}/${ceph_release}/el8/noarch
-enabled=1
-priority=2
-gpgcheck=1
-gpgkey=https://download.ceph.com/keys/release.asc
-
-[nfs-ganesha-source]
-name=NFS Ganesha source packages
-baseurl=https://download.ceph.com/nfs-ganesha/${ganesha_release}/${ceph_release}/el8/SRPMS
-enabled=0
-priority=2
-gpgcheck=1
-gpgkey=https://download.ceph.com/keys/release.asc
-EOF
-    sudo dnf config-manager --add-repo nfs-ganesha.repo
+    sudo dnf -y install ${repo}
 }

 # configure_repo_ceph() - Configure Ceph repositories
-# Usage: configure_repo_ceph <package_manager> <ceph_release> <distro_type> \
-#                            <distro_release> [<repo_type>]
-# - package_manager: apt-get or yum
-# - ceph_release: luminous, ...
-# - distro_release: 7, xenial, bionic
+# Usage: configure_repo_ceph <package_release>
+# - package_release: to override the os_RELEASE variable
 function configure_repo_ceph {
-    local package_manager="$1"
-    local ceph_release="$2"
-    local distro_release="$3"
+
+  package_release=${1:-$os_RELEASE}

    if  is_ubuntu; then
        if [[ "${TEST_MASTER}" == "True" ]]; then
            repo_file_name="/etc/apt/sources.list.d/ceph-master.list"
-            sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/ubuntu/${distro_release}/flavors/default/repo" -O ${repo_file_name}
+            sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/ubuntu/${package_release}/flavors/default/repo" -O ${repo_file_name}
        else
            wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
-            sudo apt-add-repository -y "deb https://download.ceph.com/debian-${ceph_release}/ ${distro_release} main"
+            sudo apt-add-repository -y "deb https://download.ceph.com/debian-${CEPH_RELEASE}/ $package_release main"
        fi
-        sudo ${package_manager} -y update
+        sudo apt-get -y update
    elif is_fedora; then
+        package_release="el"${package_release}
        if [[ "${TEST_MASTER}" == "True" ]]; then
            repo_file_name="/etc/yum.repos.d/ceph-master.repo"
-            sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/centos/${distro_release}/flavors/default/repo" -O ${repo_file_name}
+            sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/centos/${package_release}/flavors/default/repo" -O ${repo_file_name}
            sudo dnf config-manager --add-repo ${repo_file_name}
        else
-            dnf_add_repository_ceph ${ceph_release} ${distro_release}
+            dnf_add_repository_ceph ${CEPH_RELEASE} ${package_release}
        fi
    fi
 }
@@ -1024,37 +1010,22 @@ function cleanup_repo_ceph {
 }

 # configure_repo_nfsganesha() - Configure NFS Ganesha repositories
-# Usage: configure_repo_nfsganesha <package_manager> <ganesha_flavor> \
-#                                  <distro_type> <distro_release> [<repo_type>]
-# - package_manager: apt-get or dnf
-# - ganesha_release: 2.7, 2.8, 3.0
-# - ceph_release: ceph_luminous, ceph_nautilus
 function configure_repo_nfsganesha {
-    local package_manager="$1"
-    local ganesha_release="$2"
-    local ceph_release="$3"
-
    if is_ubuntu; then
-        # FIXME(vkmc) We need to use community ppa's because there are no builds
-        # for ubuntu bionic and ubuntu focal available for nfs-ganesha 2.7 and above
-        # Remove this when they provide the build in download.ceph.com
-        # FIXME(vkmc) Community ppa's don't provide builds for nfs-ganesha-3
-        # microversions (3.3, 3.5). Default to latest.
-        if [[ $GANESHA_RELEASE =~ V3.(3|5)-stable ]]; then
+        # NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major
+        # version; we can't use a build microversion unlike el8/el9 builds
+        if [[ $GANESHA_RELEASE =~ 3 ]]; then
            sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0
            sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0
-        elif [ $GANESHA_RELEASE == 'V2.8-stable' ]; then
-            sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-1.8
-            sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-2.8
-        elif [ $GANESHA_RELEASE == 'V2.7-stable' ]; then
-            sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-1.7
-            sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-2.7
+        elif [[ $GANESHA_RELEASE =~ 4 ]]; then
+            sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-4
+            sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-4
        else
-            die $LINENO "GANESHA_RELEASE is not supported by the Ceph plugin for Devstack"
+            die $LINENO "NFS-Ganesha $GANESHA_RELEASE is not supported by the Ceph plugin for Devstack"
        fi
-        sudo ${package_manager} -y update
+        sudo apt-get -y update
    elif is_fedora; then
-        dnf_add_repository_nfsganesha rpm-${ganesha_release} ${ceph_release}
+        dnf_add_repository_nfsganesha
    fi
 }

@@ -1062,23 +1033,7 @@ function configure_repo_nfsganesha {
 # Usage: cleanup_repo_nfsganesha
 function cleanup_repo_nfsganesha {
    if is_ubuntu; then
-        # FIXME(vkmc) We need to use community ppa's because there are no builds
-        # for ubuntu bionic and ubuntu focal available for nfs-ganesha 2.7 and above
-        # Remove this when they provide the builds in download.ceph.com
-        # FIXME(vkmc) Community ppa's don't provide builds for nfs-ganesha-3
-        # microversions (3.3, 3.5). Default to latest.
-        if [[ $GANESHA_RELEASE =~ V3.(3|5)-stable ]]; then
-            sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-libntirpc-3_0-*.list*
-            sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-nfs-ganesha-3_0-*.list*
-        elif [ $GANESHA_RELEASE == 'V2.8-stable' ]; then
-            sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-libntirpc-1_8-*.list*
-            sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-nfs-ganesha-2_8-*.list*
-        elif [ $GANESHA_RELEASE == 'V2.7-stable' ]; then
-            sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-libntirpc-1_7-*.list*
-            sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-nfs-ganesha-2_7-*.list*
-        else
-            die $LINENO "GANESHA_RELEASE is not supported by the Ceph plugin for Devstack"
-        fi
+        sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*"
    elif is_fedora; then
        sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo
    fi
@@ -1088,9 +1043,10 @@ function setup_packages_for_manila_on_ubuntu {
    CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2"

    if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
-        configure_repo_nfsganesha "apt-get" "$GANESHA_RELEASE" "$CEPH_RELEASE"
-        CEPH_PACKAGES="${CEPH_PACKAGES} libntirpc3 nfs-ganesha nfs-ganesha-ceph \
-                                        nfs-ganesha-rados-urls nfs-ganesha-vfs"
+        configure_repo_nfsganesha
+        LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}"
+        CEPH_PACKAGES="${CEPH_PACKAGES} $LIBNTIRPC_PACKAGE \
+          nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-urls nfs-ganesha-vfs"
    fi

    if python3_enabled; then
@@ -1100,31 +1056,18 @@ function setup_packages_for_manila_on_ubuntu {

 function setup_packages_for_manila_on_fedora_family {
    if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
-        # NOTE(vkmc) el7 packages work on Fedora
-        configure_repo_nfsganesha "dnf" "$GANESHA_RELEASE" "$CEPH_RELEASE"
+        configure_repo_nfsganesha
        CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph \
-                                        nfs-ganesha-rados-urls nfs-ganesha-vfs"
+           nfs-ganesha-rados-urls nfs-ganesha-vfs"
    fi
 }

 function install_ceph {

    if is_ubuntu; then
-        if ! [[ $os_CODENAME =~ (jammy|focal|xenial|bionic) ]]; then
-            die $LINENO "Supported for Ubuntu Xenial, Bionic, Focal or Jammy. \
-                         Not supported for other releases."
-        fi
-
        # NOTE(vkmc) Dependencies for setting up repos
        install_package software-properties-common

-        # NOTE(noonedeadpunk): There're no community repos for Ubuntu Jammy yet
-        #                      but Ceph Quincy is provided from default
-        #                      Ubuntu 22.04 repos.
-        if ! [[ $os_CODENAME =~ (jammy) ]]; then
-            configure_repo_ceph "apt-get" "$CEPH_RELEASE" "$os_CODENAME"
-        fi
-
        CEPH_PACKAGES="ceph libnss3-tools"
        if python3_enabled; then
            CEPH_PACKAGES="${CEPH_PACKAGES} python3-rados python3-rbd"
@@ -1150,12 +1093,14 @@ function install_ceph {

        install_package ${CEPH_PACKAGES}
    elif is_fedora; then
+        override_os_release=""
        if ! [[ $os_VENDOR =~ Fedora ]] && [[ $os_RELEASE =~ (31|32) ]]; then
            die $LINENO "Supported for Fedora 31 and 32. Not supported for other releases."
+            override_os_release="8"
        fi

        # NOTE(lyarwood) Use the py3 el8 packages on Fedora
-        configure_repo_ceph "dnf" "$CEPH_RELEASE" "el8"
+        configure_repo_ceph ${override_os_release}

        CEPH_PACKAGES="ceph"