From 563cb5deeb21815ce0c62fa30249e85e886c783a Mon Sep 17 00:00:00 2001 From: Goutham Pacha Ravi Date: Fri, 10 Mar 2023 18:44:50 -0800 Subject: [PATCH] Cleanup installation and revive cephfs-nfs job The cephfs-nfs job was turned off [1] for perma-failing. This commit adds the original non-voting job back into the check queue and fixes some installation issues: 1) use ceph "quincy" release: Ceph Pacific's end of life is 2023-06-01 [2]. The manila community thinks deployers are more likely to use quincy with the 2023.2 (bobcat) release of OpenStack. 2) run the job with centos-stream-9: There are no packages currently available for Jammy Jellyfish on download.ceph.com [3]. The OS shouldn't really matter for this CI job that is meant to test feature functionality provided by manila. At this time, we'd like to stick with builds provided by the ceph community instead of the distro since it may take a while to get bugfixes into distro builds. 3) The install script uses "nfs-ganesha" builds for ubuntu and centos hosted by the nfs-ganesha community [4]. We will not rely on the ceph community to provide the latest builds for nfs-ganesha any longer. This commit also cleans up the unnecessary condition in the ceph script file pertaining to configuring ceph packages for Jammy Jellyfish. This step wasn't doing anything. Ubuntu packages don't work at the moment and that requires some more investigation. [1] Id2ae61979505de5efb47ce90a2bac8aac2fc5484 [2] https://docs.ceph.com/en/latest/releases/ [3] https://www.spinics.net/lists/ceph-users/msg74312.html [4] https://download.nfs-ganesha.org/ Change-Id: I40dfecfbbe21b2f4b3e4efd903980b5b146c4202 Signed-off-by: Goutham Pacha Ravi --- .zuul.yaml | 22 ++- devstack/files/debs/devstack-plugin-ceph | 1 + devstack/files/rpms/devstack-plugin-ceph | 2 + devstack/lib/ceph | 175 ++++++++--------------- 4 files changed, 82 insertions(+), 118 deletions(-) create mode 100644 devstack/files/debs/devstack-plugin-ceph create mode 100644 devstack/files/rpms/devstack-plugin-ceph diff --git a/.zuul.yaml b/.zuul.yaml index 115194ab..dd3338b9 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -93,6 +93,22 @@ Runs manila tempest plugin tests with CephFS via NFS-Ganesha as a manila back end (DHSS=False) parent: manila-tempest-plugin-cephfs-nfs + nodeset: devstack-single-node-centos-9-stream + vars: + # TODO(gouthamr): some tests are disabled due to bugs + # IPv6 Tests: https://bugs.launchpad.net/manila/+bug/1998489 + # snapshot clone fs sync: https://bugs.launchpad.net/manila/+bug/1989273 + tempest_exclude_regex: "\ + (^manila_tempest_tests.tests.scenario.*IPv6.*)|\ + (^manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFS.test_write_data_to_share_created_from_snapshot)" + devstack_localrc: + MANILA_OPTGROUP_cephfsnfs_cephfs_ganesha_server_ip: "{{ hostvars[inventory_hostname]['nodepool']['private_ipv4'] }}" + CEPH_RELEASE: "quincy" + MANILA_SETUP_IPV6: false + NEUTRON_CREATE_INITIAL_NETWORKS: true + IP_VERSION: 4 + + - job: name: devstack-plugin-ceph-tempest-fedora-latest @@ -176,9 +192,9 @@ - devstack-plugin-ceph-cephfs-native: irrelevant-files: *irrelevant-files voting: false - # - devstack-plugin-ceph-cephfs-nfs: - # irrelevant-files: *irrelevant-files - # voting: false + - devstack-plugin-ceph-cephfs-nfs: + irrelevant-files: *irrelevant-files + voting: false # - devstack-plugin-ceph-tempest-fedora-latest # - devstack-plugin-ceph-multinode-tempest-py3 # - devstack-plugin-ceph-multinode-tempest-cephadm: diff --git a/devstack/files/debs/devstack-plugin-ceph b/devstack/files/debs/devstack-plugin-ceph new file mode 100644 index 00000000..0dee61c0 --- /dev/null +++ b/devstack/files/debs/devstack-plugin-ceph @@ -0,0 +1 @@ +xfsprogs \ No newline at end of file diff --git a/devstack/files/rpms/devstack-plugin-ceph b/devstack/files/rpms/devstack-plugin-ceph new file mode 100644 index 00000000..2d67a35e --- /dev/null +++ b/devstack/files/rpms/devstack-plugin-ceph @@ -0,0 +1,2 @@ +xfsprogs +dbus-tools \ No newline at end of file diff --git a/devstack/lib/ceph b/devstack/lib/ceph index 9b37b617..a054d9ac 100755 --- a/devstack/lib/ceph +++ b/devstack/lib/ceph @@ -30,7 +30,19 @@ TEST_MASTER=$(trueorfalse False TEST_MASTER) CEPH_RELEASE=${CEPH_RELEASE:-pacific} -GANESHA_RELEASE=${GANESHA_RELEASE:-V3.5-stable} +GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'} +# Remove "v" and "-stable" prefix/suffix tags +GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//") + +if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then + # default ganesha release based on ceph release + case $CEPH_RELEASE in + pacific) + GANESHA_RELEASE='3.5' ;; + *) + GANESHA_RELEASE='4.0' ;; + esac +fi # Deploy a Ceph demo container instead of a non-containerized version CEPH_CONTAINERIZED=$(trueorfalse False CEPH_CONTAINERIZED) @@ -273,17 +285,13 @@ function _undefine_virsh_secret { # check_os_support_ceph() - Check if the OS provides a decent version of Ceph function check_os_support_ceph { - if [[ ! ${DISTRO} =~ (jammy|focal|bionic|xenial|f31|f32|f33|f34|rhel8) ]]; then - echo "WARNING: your distro $DISTRO does not provide \ - (at least) the Luminous release. \ - Please use Ubuntu Xenial, Ubuntu Bionic, Ubuntu Focal, Ubuntu Jammy, \ - Fedora 31-34 or CentOS Stream 8." + if [[ ! ${DISTRO} =~ (jammy|focal|bionic|xenial|f31|f32|f33|f34|rhel8|rhel9) ]]; then + echo "WARNING: devstack-plugin-ceph hasn't been tested with $DISTRO. \ + Set FORCE_CEPH_INSTALL=yes in your local.conf if you'd like to \ + attempt installation anyway." if [[ "$FORCE_CEPH_INSTALL" != "yes" ]]; then - die $LINENO "If you wish to install Ceph on this distribution \ - anyway run with FORCE_CEPH_INSTALL=yes, \ - this assumes that YOU will setup the proper repositories" + die $LINENO "Not proceeding with install." fi - NO_UPDATE_REPOS=False fi if [[ ! $INIT_SYSTEM == 'systemd' ]]; then @@ -893,7 +901,7 @@ function configure_ceph_cinder { sudo rbd pool init ${CINDER_CEPH_POOL} } -# install_ceph() - Collect source and prepare +# install_ceph_remote() - Collect source and prepare function install_ceph_remote { install_package ceph-common # ceph-common in Bionic (18.04) installs only the python2 variants of @@ -920,12 +928,12 @@ function install_ceph_remote { function dnf_add_repository_ceph { local ceph_release=$1 - local distro_release=$2 + local package_release=$2 cat > ceph.repo < nfs-ganesha.repo < \ -# [] -# - package_manager: apt-get or yum -# - ceph_release: luminous, ... -# - distro_release: 7, xenial, bionic +# Usage: configure_repo_ceph +# - package_release: to override the os_RELEASE variable function configure_repo_ceph { - local package_manager="$1" - local ceph_release="$2" - local distro_release="$3" + + package_release=${1:-$os_RELEASE} if is_ubuntu; then if [[ "${TEST_MASTER}" == "True" ]]; then repo_file_name="/etc/apt/sources.list.d/ceph-master.list" - sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/ubuntu/${distro_release}/flavors/default/repo" -O ${repo_file_name} + sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/ubuntu/${package_release}/flavors/default/repo" -O ${repo_file_name} else wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add - - sudo apt-add-repository -y "deb https://download.ceph.com/debian-${ceph_release}/ ${distro_release} main" + sudo apt-add-repository -y "deb https://download.ceph.com/debian-${CEPH_RELEASE}/ $package_release main" fi - sudo ${package_manager} -y update + sudo apt-get -y update elif is_fedora; then + package_release="el"${package_release} if [[ "${TEST_MASTER}" == "True" ]]; then repo_file_name="/etc/yum.repos.d/ceph-master.repo" - sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/centos/${distro_release}/flavors/default/repo" -O ${repo_file_name} + sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/centos/${package_release}/flavors/default/repo" -O ${repo_file_name} sudo dnf config-manager --add-repo ${repo_file_name} else - dnf_add_repository_ceph ${ceph_release} ${distro_release} + dnf_add_repository_ceph ${CEPH_RELEASE} ${package_release} fi fi } @@ -1024,37 +1010,22 @@ function cleanup_repo_ceph { } # configure_repo_nfsganesha() - Configure NFS Ganesha repositories -# Usage: configure_repo_nfsganesha \ -# [] -# - package_manager: apt-get or dnf -# - ganesha_release: 2.7, 2.8, 3.0 -# - ceph_release: ceph_luminous, ceph_nautilus function configure_repo_nfsganesha { - local package_manager="$1" - local ganesha_release="$2" - local ceph_release="$3" - if is_ubuntu; then - # FIXME(vkmc) We need to use community ppa's because there are no builds - # for ubuntu bionic and ubuntu focal available for nfs-ganesha 2.7 and above - # Remove this when they provide the build in download.ceph.com - # FIXME(vkmc) Community ppa's don't provide builds for nfs-ganesha-3 - # microversions (3.3, 3.5). Default to latest. - if [[ $GANESHA_RELEASE =~ V3.(3|5)-stable ]]; then + # NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major + # version; we can't use a build microversion unlike el8/el9 builds + if [[ $GANESHA_RELEASE =~ 3 ]]; then sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0 sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0 - elif [ $GANESHA_RELEASE == 'V2.8-stable' ]; then - sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-1.8 - sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-2.8 - elif [ $GANESHA_RELEASE == 'V2.7-stable' ]; then - sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-1.7 - sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-2.7 + elif [[ $GANESHA_RELEASE =~ 4 ]]; then + sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-4 + sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-4 else - die $LINENO "GANESHA_RELEASE is not supported by the Ceph plugin for Devstack" + die $LINENO "NFS-Ganesha $GANESHA_RELEASE is not supported by the Ceph plugin for Devstack" fi - sudo ${package_manager} -y update + sudo apt-get -y update elif is_fedora; then - dnf_add_repository_nfsganesha rpm-${ganesha_release} ${ceph_release} + dnf_add_repository_nfsganesha fi } @@ -1062,23 +1033,7 @@ function configure_repo_nfsganesha { # Usage: cleanup_repo_nfsganesha function cleanup_repo_nfsganesha { if is_ubuntu; then - # FIXME(vkmc) We need to use community ppa's because there are no builds - # for ubuntu bionic and ubuntu focal available for nfs-ganesha 2.7 and above - # Remove this when they provide the builds in download.ceph.com - # FIXME(vkmc) Community ppa's don't provide builds for nfs-ganesha-3 - # microversions (3.3, 3.5). Default to latest. - if [[ $GANESHA_RELEASE =~ V3.(3|5)-stable ]]; then - sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-libntirpc-3_0-*.list* - sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-nfs-ganesha-3_0-*.list* - elif [ $GANESHA_RELEASE == 'V2.8-stable' ]; then - sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-libntirpc-1_8-*.list* - sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-nfs-ganesha-2_8-*.list* - elif [ $GANESHA_RELEASE == 'V2.7-stable' ]; then - sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-libntirpc-1_7-*.list* - sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-nfs-ganesha-2_7-*.list* - else - die $LINENO "GANESHA_RELEASE is not supported by the Ceph plugin for Devstack" - fi + sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*" elif is_fedora; then sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo fi @@ -1088,9 +1043,10 @@ function setup_packages_for_manila_on_ubuntu { CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2" if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then - configure_repo_nfsganesha "apt-get" "$GANESHA_RELEASE" "$CEPH_RELEASE" - CEPH_PACKAGES="${CEPH_PACKAGES} libntirpc3 nfs-ganesha nfs-ganesha-ceph \ - nfs-ganesha-rados-urls nfs-ganesha-vfs" + configure_repo_nfsganesha + LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}" + CEPH_PACKAGES="${CEPH_PACKAGES} $LIBNTIRPC_PACKAGE \ + nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-urls nfs-ganesha-vfs" fi if python3_enabled; then @@ -1100,31 +1056,18 @@ function setup_packages_for_manila_on_ubuntu { function setup_packages_for_manila_on_fedora_family { if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then - # NOTE(vkmc) el7 packages work on Fedora - configure_repo_nfsganesha "dnf" "$GANESHA_RELEASE" "$CEPH_RELEASE" + configure_repo_nfsganesha CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph \ - nfs-ganesha-rados-urls nfs-ganesha-vfs" + nfs-ganesha-rados-urls nfs-ganesha-vfs" fi } function install_ceph { if is_ubuntu; then - if ! [[ $os_CODENAME =~ (jammy|focal|xenial|bionic) ]]; then - die $LINENO "Supported for Ubuntu Xenial, Bionic, Focal or Jammy. \ - Not supported for other releases." - fi - # NOTE(vkmc) Dependencies for setting up repos install_package software-properties-common - # NOTE(noonedeadpunk): There're no community repos for Ubuntu Jammy yet - # but Ceph Quincy is provided from default - # Ubuntu 22.04 repos. - if ! [[ $os_CODENAME =~ (jammy) ]]; then - configure_repo_ceph "apt-get" "$CEPH_RELEASE" "$os_CODENAME" - fi - CEPH_PACKAGES="ceph libnss3-tools" if python3_enabled; then CEPH_PACKAGES="${CEPH_PACKAGES} python3-rados python3-rbd" @@ -1150,12 +1093,14 @@ function install_ceph { install_package ${CEPH_PACKAGES} elif is_fedora; then + override_os_release="" if ! [[ $os_VENDOR =~ Fedora ]] && [[ $os_RELEASE =~ (31|32) ]]; then die $LINENO "Supported for Fedora 31 and 32. Not supported for other releases." + override_os_release="8" fi # NOTE(lyarwood) Use the py3 el8 packages on Fedora - configure_repo_ceph "dnf" "$CEPH_RELEASE" "el8" + configure_repo_ceph ${override_os_release} CEPH_PACKAGES="ceph"