From fac55d15453d7666a1a63d53d128bf826425817d Mon Sep 17 00:00:00 2001 From: ashrod98 Date: Mon, 16 Oct 2023 19:53:44 +0000 Subject: [PATCH] Remote Ceph with cephadm Add podman ceph-common and jq as part of preinstall dependency. Add REMOTE_CEPH capabilities to CEPHADM deployment. Removed set_min_client only if cinder is enabled, this should be set in any case. Get FSID from ceph.conf in /etc/ceph to avoid unnecessary override. Update paste-deploy workaround to be used always. Part of an effort to test multinode deployments with cephadm. Pinned tempest-py3-base to single-node-jammy. Added cephadm deploy to tempest-py3 job. Needed-By: I5162815b66d3f3e8cf8c1e246b61b0ea06c1a270 Change-Id: I84249ae268dfe00a112c67e5170b679acb318a25 --- .zuul.yaml | 81 ++++++------------------ devstack/files/debs/devstack-plugin-ceph | 7 +- devstack/files/rpms/devstack-plugin-ceph | 5 +- devstack/lib/cephadm | 56 ++++++++-------- devstack/override-defaults | 1 + devstack/plugin.sh | 11 ++-- 6 files changed, 62 insertions(+), 99 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 2ecc11cb..9fe196d2 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -2,9 +2,7 @@ name: devstack-plugin-ceph-tempest-py3-base abstract: true parent: tempest-full-py3 - # TODO: Remove the nodeset pinning to focal once below bug is fixed - # https://bugs.launchpad.net/devstack-plugin-ceph/+bug/1996628 - nodeset: openstack-single-node-focal + nodeset: openstack-single-node-jammy description: | Base integration tests that runs with the ceph devstack plugin and py3. Former names for this job where: @@ -48,19 +46,8 @@ This job enable the multiattach feature enable from stein on. vars: devstack_localrc: - ENABLE_VOLUME_MULTIATTACH: true - CEPH_RELEASE: "pacific" - -- job: - name: devstack-plugin-ceph-tempest-cephadm - parent: devstack-plugin-ceph-tempest-py3-base - description: | - Integration tests that runs with the ceph devstack plugin and py3. - The ceph cluster is deployed using cephadm - vars: - tempest_concurrency: 1 - devstack_localrc: - CEPHADM_DEPLOY: true + DISABLE_CEPHADM_POST_DEPLOY: True + CEPHADM_DEPLOY: True - job: name: devstack-plugin-ceph-compute-local-ephemeral @@ -96,50 +83,29 @@ Runs manila tempest plugin tests with CephFS via NFS-Ganesha as a manila back end (DHSS=False) parent: manila-tempest-plugin-cephfs-nfs - nodeset: devstack-single-node-centos-9-stream - vars: - # TODO(gouthamr): some tests are disabled due to bugs - # IPv6 Tests: https://bugs.launchpad.net/manila/+bug/1998489 - # snapshot clone fs sync: https://bugs.launchpad.net/manila/+bug/1989273 - tempest_exclude_regex: "\ - (^manila_tempest_tests.tests.scenario.*IPv6.*)|\ - (^manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFS.test_write_data_to_share_created_from_snapshot)" - devstack_localrc: - MANILA_OPTGROUP_cephfsnfs_cephfs_ganesha_server_ip: "{{ hostvars[inventory_hostname]['nodepool']['private_ipv4'] }}" - CEPH_RELEASE: "quincy" - MANILA_SETUP_IPV6: false - NEUTRON_CREATE_INITIAL_NETWORKS: true - IP_VERSION: 4 - - - -- job: - name: devstack-plugin-ceph-tempest-fedora-latest - parent: devstack-plugin-ceph-tempest-py3 - description: | - Integration tests that runs with the ceph devstack plugin on Fedora. - nodeset: devstack-single-node-fedora-latest - voting: false - job: name: devstack-plugin-ceph-multinode-tempest-py3 parent: tempest-multinode-full-py3 description: | Integration tests that runs the ceph device plugin across multiple - nodes on py3. - # TODO: Remove the nodeset pinning to focal once below bug is fixed - # https://bugs.launchpad.net/devstack-plugin-ceph/+bug/1996628 - nodeset: openstack-two-node-focal + nodes on py3. The Ceph deployment strategy used by this job is Cephadm. required-projects: - openstack/cinder-tempest-plugin - openstack/devstack-plugin-ceph timeout: 10800 - voting: false vars: + configure_swap_size: 8192 + tempest_concurrency: 3 devstack_localrc: ENABLE_FILE_INJECTION: false ENABLE_VOLUME_MULTIATTACH: true - TEMPEST_RUN_VALIDATION: false + TEMPEST_RUN_VALIDATION: true + USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION: false + CEPHADM_DEPLOY: True + DISABLE_CEPHADM_POST_DEPLOY: True + MYSQL_REDUCE_MEMORY: True + REMOTE_CEPH: False CINDER_CEPH_UUID: d531d2d4-3937-429c-b0c2-658fe41e82aa devstack_plugins: devstack-plugin-ceph: https://opendev.org/openstack/devstack-plugin-ceph @@ -155,7 +121,8 @@ group-vars: subnode: devstack_localrc: - REMOTE_CEPH: true + REMOTE_CEPH: True + CEPHADM_DEPLOY: True CINDER_CEPH_UUID: d531d2d4-3937-429c-b0c2-658fe41e82aa - job: @@ -169,19 +136,6 @@ devstack_localrc: TEST_MASTER: true -- job: - name: devstack-plugin-ceph-multinode-tempest-cephadm - parent: devstack-plugin-ceph-multinode-tempest-py3 - description: | - Integration tests that runs the ceph device plugin across multiple - nodes on py3. - The ceph deployment strategy used by this job is cephadm. - vars: - devstack_localrc: - USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION: false - CEPHADM_DEPLOY: true - tempest_concurrency: 1 - - project-template: name: devstack-plugin-ceph-tempest-jobs description: | @@ -192,6 +146,7 @@ voting: false - devstack-plugin-ceph-tempest-cephadm: voting: false + - devstack-plugin-ceph-multinode-tempest-py3 - devstack-plugin-ceph-cephfs-native: irrelevant-files: *irrelevant-files voting: false @@ -204,9 +159,9 @@ # voting: false # - devstack-plugin-ceph-master-tempest: # voting: false - # gate: - # jobs: - # - devstack-plugin-ceph-tempest-py3 + gate: + jobs: + - devstack-plugin-ceph-tempest-py3 - project: templates: diff --git a/devstack/files/debs/devstack-plugin-ceph b/devstack/files/debs/devstack-plugin-ceph index 0dee61c0..73bc9b92 100644 --- a/devstack/files/debs/devstack-plugin-ceph +++ b/devstack/files/debs/devstack-plugin-ceph @@ -1 +1,6 @@ -xfsprogs \ No newline at end of file +xfsprogs +qemu-block-extra +catatonit +podman +jq +ceph-common diff --git a/devstack/files/rpms/devstack-plugin-ceph b/devstack/files/rpms/devstack-plugin-ceph index 2d67a35e..1b806da7 100644 --- a/devstack/files/rpms/devstack-plugin-ceph +++ b/devstack/files/rpms/devstack-plugin-ceph @@ -1,2 +1,5 @@ xfsprogs -dbus-tools \ No newline at end of file +dbus-tools +podman +jq +ceph-common diff --git a/devstack/lib/cephadm b/devstack/lib/cephadm index 8b5e685e..d6d44f44 100755 --- a/devstack/lib/cephadm +++ b/devstack/lib/cephadm @@ -31,7 +31,11 @@ DISABLE_CEPHADM_POST_DEPLOY=${DISABLE_CEPHADM_POST_DEPLOY:-False} ATTEMPTS=30 CONTAINER_IMAGE=${CONTAINER_IMAGE:-'quay.io/ceph/ceph:v17.2.3'} DEVICES=() -FSID=$(uuidgen) +if [[ "$REMOTE_CEPH" = "False" ]]; then + FSID=$(uuidgen) +else + FSID=$(cat $CEPH_CONFIG | grep fsid | awk 'BEGIN { RS = "fsid = "} ; { print $0 }' - ) +fi KEY_EXPORT_DIR="/etc/ceph" KEYS=("client.openstack") # at least the client.openstack default key should be created MIN_OSDS=1 @@ -117,33 +121,19 @@ function export_spec { echo "Ceph cluster config exported: $EXPORT" } -# Pre-install ceph: install podman -function _install_podman { - # FIXME(vkmc) Check required for Ubuntu 20.04 LTS (current CI node) - # Remove when our CI is pushed to the next LTS version - if ! command -v podman &> /dev/null; then - if [[ $os_CODENAME =~ (focal) ]]; then - echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /" \ - | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list - curl -L "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/Release.key" \ - | sudo apt-key add - - sudo apt-get update - sudo apt-get -y upgrade - fi - install_package podman - fi -} - # Pre-install ceph: install required dependencies function install_deps { - install_package jq ceph-common - _install_podman - install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests + if [[ "$REMOTE_CEPH" == "False" ]]; then + install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests + fi } # Pre-install ceph: get cephadm binary function get_cephadm { - curl -O https://raw.githubusercontent.com/ceph/ceph/"$CEPH_RELEASE"/src/cephadm/cephadm + # NOTE(gouthamr): cephadm binary here is a python executable, and the + # $os_PACKAGE ("rpm") or $os_release (el9) doesn't really matter. There is + # no ubuntu/debian equivalent being published by the ceph community. + curl -O https://download.ceph.com/rpm-${CEPH_RELEASE}/el9/noarch/cephadm $SUDO mv cephadm $TARGET_BIN $SUDO chmod +x $TARGET_BIN/cephadm echo "[GET CEPHADM] cephadm is now available" @@ -176,7 +166,7 @@ EOF function start_ceph { cluster=$(sudo cephadm ls | jq '.[]' | jq 'select(.name | test("^mon*")).fsid') if [ -z "$cluster" ]; then - $SUDO $CEPHADM --image "$CONTAINER_IMAGE" \ + $SUDO "$CEPHADM" --image "$CONTAINER_IMAGE" \ bootstrap \ --fsid $FSID \ --config "$BOOTSTRAP_CONFIG" \ @@ -234,7 +224,7 @@ function add_osds { # let's add some osds if [ -z "$DEVICES" ]; then echo "Using ALL available devices" - $SUDO $CEPHADM shell ceph orch apply osd --all-available-devices + $SUDO "$CEPHADM" shell ceph orch apply osd --all-available-devices else for item in "${DEVICES[@]}"; do echo "Creating osd $item on node $HOSTNAME" @@ -244,7 +234,7 @@ function add_osds { fi while [ "$ATTEMPTS" -ne 0 ]; do - num_osds=$($SUDO $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ + num_osds=$($SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds') if [ "$num_osds" -ge "$MIN_OSDS" ]; then break; @@ -300,6 +290,7 @@ function _create_key { --keyring $CEPH_KEYRING -- ceph auth get-or-create "$name" mgr "allow rw" mon "allow r" osd "$osd_caps" \ -o "$KEY_EXPORT_DIR/ceph.$name.keyring" + $SUDO chown ${STACK_USER}:$(id -g -n $whoami) \ ${CEPH_CONF_DIR}/ceph.$name.keyring } @@ -642,7 +633,6 @@ function configure_ceph { if is_ceph_enabled_for_service cinder; then POOLS+=($CINDER_CEPH_POOL) KEYS+=("client.$CINDER_CEPH_USER") - set_min_client_version fi if is_ceph_enabled_for_service c-bak; then @@ -659,8 +649,10 @@ function configure_ceph { [ "$ENABLE_CEPH_RGW" == "True" ] && SERVICES+=('rgw') enable_services - add_pools - create_keys + if [[ "$REMOTE_CEPH" = "False" ]]; then + add_pools + create_keys + fi client_config import_libvirt_secret_ceph @@ -677,8 +669,12 @@ function configure_ceph_manila { function cleanup_ceph { # Cleanup the service. - stop_ceph - delete_osd_dev + if [[ "$REMOTE_CEPH" == "True" ]]; then + echo "Remote Ceph cluster, skipping stop_ceph and delete_osd_dev" + else + stop_ceph + delete_osd_dev + fi # purge ceph config file and keys $SUDO rm -f ${CEPH_CONF_DIR}/* if is_ceph_enabled_for_service nova; then diff --git a/devstack/override-defaults b/devstack/override-defaults index 18afcd6a..aa80ef17 100644 --- a/devstack/override-defaults +++ b/devstack/override-defaults @@ -20,3 +20,4 @@ if [[ $ENABLE_CEPH_CINDER == "True" ]]; then fi CEPHADM_DEPLOY=$(trueorfalse False CEPHADM_DEPLOY) +REMOTE_CEPH=$(trueorfalse False REMOTE_CEPH) diff --git a/devstack/plugin.sh b/devstack/plugin.sh index c6063f17..b415f6ec 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -40,10 +40,14 @@ elif [[ "$1" == "stack" && "$2" == "pre-install" ]]; then fi fi elif [[ "$1" == "stack" && "$2" == "install" ]]; then - if [[ "$CEPHADM_DEPLOY" = "True" ]]; then + if [[ "$CEPHADM_DEPLOY" = "True" && "$REMOTE_CEPH" = "False" ]]; then # Perform installation of service source echo_summary "[cephadm] Installing ceph" install_ceph + set_min_client_version + elif [[ "$CEPHADM_DEPLOY" = "True" && "$REMOTE_CEPH" = "True" ]]; then + echo "[CEPHADM] Remote Ceph: Skipping install" + get_cephadm else # FIXME(melwitt): This is a hack to get around a namespacing issue with # Paste and PasteDeploy. For stable/queens, we use the Pike UCA packages @@ -53,9 +57,8 @@ elif [[ "$1" == "stack" && "$2" == "install" ]]; then # newer version of it, while python-pastedeploy remains. The mismatch # between the install path of paste and paste.deploy causes Keystone to # fail to start, with "ImportError: cannot import name deploy." - if [[ "$TARGET_BRANCH" == stable/queens || "$TARGET_BRANCH" == master ]]; then - pip_install -U --force PasteDeploy - fi + pip_install -U --force PasteDeploy + install_package python-is-python3 fi elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then if [[ "$CEPHADM_DEPLOY" = "True" ]]; then