From 1c192e89fcc169e8731f80b281780b7dac8a0aaa Mon Sep 17 00:00:00 2001 From: Ashley Rodriguez Date: Tue, 7 Mar 2023 14:54:34 +0000 Subject: [PATCH] [WIP][DNM] Remote Ceph with cephadm Add podman as part of preinstall dependency. Add REMOTE_CEPH capabilities to CEPHADM deployment. Part of an effort to get multinode ceph with manila testing to run locally, and hopefully on CI at a future date. Change-Id: I84249ae268dfe00a112c67e5170b679acb318a25 --- .zuul.yaml | 111 +++-------------------- devstack/files/rpms/devstack-plugin-ceph | 3 +- devstack/lib/cephadm | 72 +++++++-------- devstack/override-defaults | 1 + devstack/plugin.sh | 4 +- 5 files changed, 52 insertions(+), 139 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 2e7b353d..1423f0e3 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -36,18 +36,6 @@ (^tempest\.(api|scenario\.test_encrypted_cinder_volumes|scenario\.test_volume|scenario\.test_shelve_instance)|(^cinder_tempest_plugin)) tempest_test_blacklist: '{{ ansible_user_dir }}/{{ zuul.projects["opendev.org/openstack/devstack-plugin-ceph"].src_dir }}/tempest_skiplist.txt' -- job: - name: devstack-plugin-ceph-tempest-ubuntu - parent: devstack-plugin-ceph-tempest-py3-base - description: | - Integration tests that runs with the ceph devstack plugin using ceph - packages from the distro. - - This job enable the multiattach feature enable from stein on. - vars: - devstack_localrc: - ENABLE_VOLUME_MULTIATTACH: true - - job: name: devstack-plugin-ceph-tempest-py3 parent: devstack-plugin-ceph-tempest-py3-base @@ -59,65 +47,6 @@ DISABLE_CEPHADM_POST_DEPLOY: True CEPHADM_DEPLOY: true -- job: - name: devstack-plugin-ceph-compute-local-ephemeral - parent: devstack-plugin-ceph-tempest-py3-base - description: | - Integration tests that runs with the ceph devstack plugin and py3. - - This job does not configure Nova to use rbd for ephemeral storage. It - also enables the direct download of images via rbd into the local - imagecache for Nova. - vars: - devstack_local_conf: - post-config: - $NOVA_CONF: - glance: - enable_rbd_download: True - rbd_user: glance - rbd_ceph_conf: /etc/ceph/ceph.conf - rbd_pool: images - libvirt: - images_type: default - -- job: - name: devstack-plugin-ceph-cephfs-native - description: | - Runs manila tempest plugin tests with Native CephFS as a manila back - end (DHSS=False) - parent: manila-tempest-plugin-cephfs-native-cephadm - -- job: - name: devstack-plugin-ceph-cephfs-nfs - description: | - Runs manila tempest plugin tests with CephFS via NFS-Ganesha as a manila - back end (DHSS=False) - parent: manila-tempest-plugin-cephfs-nfs - nodeset: devstack-single-node-centos-9-stream - vars: - # TODO(gouthamr): some tests are disabled due to bugs - # IPv6 Tests: https://bugs.launchpad.net/manila/+bug/1998489 - # snapshot clone fs sync: https://bugs.launchpad.net/manila/+bug/1989273 - tempest_exclude_regex: "\ - (^manila_tempest_tests.tests.scenario.*IPv6.*)|\ - (^manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFS.test_write_data_to_share_created_from_snapshot)" - devstack_localrc: - MANILA_OPTGROUP_cephfsnfs_cephfs_ganesha_server_ip: "{{ hostvars[inventory_hostname]['nodepool']['private_ipv4'] }}" - CEPH_RELEASE: "quincy" - MANILA_SETUP_IPV6: false - NEUTRON_CREATE_INITIAL_NETWORKS: true - IP_VERSION: 4 - - - -- job: - name: devstack-plugin-ceph-tempest-fedora-latest - parent: devstack-plugin-ceph-tempest-py3 - description: | - Integration tests that runs with the ceph devstack plugin on Fedora. - nodeset: devstack-single-node-fedora-latest - voting: false - - job: name: devstack-plugin-ceph-multinode-tempest-py3 parent: tempest-multinode-full-py3 @@ -130,6 +59,8 @@ timeout: 10800 voting: false vars: + configure_swap_size: 8192 + tempest_concurrency: 3 devstack_localrc: ENABLE_FILE_INJECTION: false ENABLE_VOLUME_MULTIATTACH: true @@ -149,20 +80,9 @@ group-vars: subnode: devstack_localrc: - REMOTE_CEPH: true + REMOTE_CEPH: True CINDER_CEPH_UUID: d531d2d4-3937-429c-b0c2-658fe41e82aa -- job: - name: devstack-plugin-ceph-master-tempest - parent: devstack-plugin-ceph-tempest-py3-base - description: | - Integration tests that runs with the ceph devstack plugin - using Ceph master as the backend. - branches: master - vars: - devstack_localrc: - TEST_MASTER: true - - job: name: devstack-plugin-ceph-multinode-tempest-cephadm parent: devstack-plugin-ceph-multinode-tempest-py3 @@ -173,8 +93,16 @@ vars: devstack_localrc: USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION: false - CEPHADM_DEPLOY: true + DISABLE_CEPHADM_POST_DEPLOY: True + CEPHADM_DEPLOY: True + MYSQL_REDUCE_MEMORY: True + REMOTE_CEPH: False tempest_concurrency: 1 + group-vars: + subnode: + devstack_localrc: + CEPHADM_DEPLOY: True + REMOTE_CEPH: True - project-template: name: devstack-plugin-ceph-tempest-jobs @@ -183,20 +111,11 @@ check: jobs: - devstack-plugin-ceph-tempest-py3 - - devstack-plugin-ceph-tempest-ubuntu: + - devstack-plugin-ceph-multinode-tempest-py3: voting: false - - devstack-plugin-ceph-cephfs-native: - irrelevant-files: *irrelevant-files + - devstack-plugin-ceph-multinode-tempest-cephadm: voting: false - - devstack-plugin-ceph-cephfs-nfs: - irrelevant-files: *irrelevant-files - voting: false - # - devstack-plugin-ceph-tempest-fedora-latest - # - devstack-plugin-ceph-multinode-tempest-py3 - # - devstack-plugin-ceph-multinode-tempest-cephadm: - # voting: false - # - devstack-plugin-ceph-master-tempest: - # voting: false + gate: jobs: - devstack-plugin-ceph-tempest-py3 diff --git a/devstack/files/rpms/devstack-plugin-ceph b/devstack/files/rpms/devstack-plugin-ceph index 2d67a35e..6f9079e7 100644 --- a/devstack/files/rpms/devstack-plugin-ceph +++ b/devstack/files/rpms/devstack-plugin-ceph @@ -1,2 +1,3 @@ xfsprogs -dbus-tools \ No newline at end of file +dbus-tools +podman diff --git a/devstack/lib/cephadm b/devstack/lib/cephadm index 8b5e685e..2ffffe15 100755 --- a/devstack/lib/cephadm +++ b/devstack/lib/cephadm @@ -98,7 +98,7 @@ NOVA_CEPH_POOL=${NOVA_CEPH_POOL:-vms} function set_debug { if [ "$DEBUG" -eq 1 ]; then echo "[CEPHADM] Enabling Debug mode" - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph config set mgr mgr/cephadm/log_to_cluster_level debug echo "[CEPHADM] See debug logs running: ceph -W cephadm --watch-debug" fi @@ -106,39 +106,25 @@ function set_debug { # Admin: check ceph cluster status function check_cluster_status { - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph -s -f json-pretty } # Admin: export ceph cluster config spec function export_spec { - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph orch ls --export > "$EXPORT" echo "Ceph cluster config exported: $EXPORT" } -# Pre-install ceph: install podman -function _install_podman { - # FIXME(vkmc) Check required for Ubuntu 20.04 LTS (current CI node) - # Remove when our CI is pushed to the next LTS version - if ! command -v podman &> /dev/null; then - if [[ $os_CODENAME =~ (focal) ]]; then - echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /" \ - | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list - curl -L "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/Release.key" \ - | sudo apt-key add - - sudo apt-get update - sudo apt-get -y upgrade - fi - install_package podman - fi -} - # Pre-install ceph: install required dependencies function install_deps { - install_package jq ceph-common - _install_podman - install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests + if [[ "$REMOTE_CEPH" == "True" ]]; then + install_package ceph-common + else + install_package jq ceph-common + install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests + fi } # Pre-install ceph: get cephadm binary @@ -149,7 +135,7 @@ function get_cephadm { echo "[GET CEPHADM] cephadm is now available" if [ -z "$CEPHADM" ]; then - CEPHADM=${TARGET_BIN}/cephadm + CEPHADM="${SUDO} ${TARGET_BIN}/cephadm" fi } @@ -176,7 +162,7 @@ EOF function start_ceph { cluster=$(sudo cephadm ls | jq '.[]' | jq 'select(.name | test("^mon*")).fsid') if [ -z "$cluster" ]; then - $SUDO $CEPHADM --image "$CONTAINER_IMAGE" \ + $CEPHADM --image "$CONTAINER_IMAGE" \ bootstrap \ --fsid $FSID \ --config "$BOOTSTRAP_CONFIG" \ @@ -234,17 +220,17 @@ function add_osds { # let's add some osds if [ -z "$DEVICES" ]; then echo "Using ALL available devices" - $SUDO $CEPHADM shell ceph orch apply osd --all-available-devices + $CEPHADM shell ceph orch apply osd --all-available-devices else for item in "${DEVICES[@]}"; do echo "Creating osd $item on node $HOSTNAME" - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph orch daemon add osd "$HOSTNAME:$item" done fi while [ "$ATTEMPTS" -ne 0 ]; do - num_osds=$($SUDO $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ + num_osds=$($CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds') if [ "$num_osds" -ge "$MIN_OSDS" ]; then break; @@ -263,12 +249,12 @@ function add_pools { [ "${#POOLS[@]}" -eq 0 ] && return; for pool in "${POOLS[@]}"; do - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph osd pool create "$pool" "$DEFAULT_PG_NUM" \ "$DEFAULT_PGP_NUM" replicated --autoscale-mode on # set the application to the pool (which also means rbd init the pool) - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph osd pool application enable "$pool" rbd done } @@ -296,7 +282,7 @@ function _create_key { osd_caps="allow class-read object_prefix rbd_children, $caps" fi - $SUDO "$CEPHADM" shell -v "$KEY_EXPORT_DIR:$KEY_EXPORT_DIR" --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell -v "$KEY_EXPORT_DIR:$KEY_EXPORT_DIR" --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph auth get-or-create "$name" mgr "allow rw" mon "allow r" osd "$osd_caps" \ -o "$KEY_EXPORT_DIR/ceph.$name.keyring" @@ -318,7 +304,7 @@ function cephfs_config { # - $FSNAME.FSNAME.data # - $FSNAME.FSNAME.meta # and the mds daemon is deployed - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph fs volume create "$FSNAME" } @@ -326,7 +312,7 @@ function cephfs_config { function ceph_nfs_config { # (fpantano) TODO: Build an ingress daemon on top of this echo "[CEPHADM] Deploy nfs.$FSNAME backend" - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph orch apply nfs \ "$FSNAME" --placement="$HOSTNAME" --port $NFS_PORT } @@ -368,7 +354,7 @@ function set_config_key { local section=$1 local key=$2 local value=$3 - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ ceph config set ${section} ${key} ${value} } @@ -407,7 +393,7 @@ function configure_rgw_ceph_section { function rgw { configure_ceph_embedded_rgw - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph orch apply rgw default default default default \ "--placement=$HOSTNAME count:1" --port "$RGW_PORT" @@ -490,7 +476,7 @@ function stop_ceph { if ! [ -x "$CEPHADM" ]; then get_cephadm - CEPHADM=${TARGET_BIN}/cephadm + CEPHADM="${SUDO} ${TARGET_BIN}/cephadm" fi cluster_deleted=0 @@ -589,7 +575,7 @@ function config_nova { function set_min_client_version { if [ ! -z "$CEPH_MIN_CLIENT_VERSION" ]; then - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph osd set-require-min-compat-client ${CEPH_MIN_CLIENT_VERSION} fi } @@ -677,8 +663,12 @@ function configure_ceph_manila { function cleanup_ceph { # Cleanup the service. - stop_ceph - delete_osd_dev + if [[ "$REMOTE_CEPH" == "True" ]]; then + echo "Remote Ceph cluster, skip stop_ceph and delete_osd_dev" + else + stop_ceph + delete_osd_dev + fi # purge ceph config file and keys $SUDO rm -f ${CEPH_CONF_DIR}/* if is_ceph_enabled_for_service nova; then @@ -687,10 +677,10 @@ function cleanup_ceph { } function disable_cephadm { - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph orch set backend - $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph mgr module disable cephadm } diff --git a/devstack/override-defaults b/devstack/override-defaults index 18afcd6a..aa80ef17 100644 --- a/devstack/override-defaults +++ b/devstack/override-defaults @@ -20,3 +20,4 @@ if [[ $ENABLE_CEPH_CINDER == "True" ]]; then fi CEPHADM_DEPLOY=$(trueorfalse False CEPHADM_DEPLOY) +REMOTE_CEPH=$(trueorfalse False REMOTE_CEPH) diff --git a/devstack/plugin.sh b/devstack/plugin.sh index aa7cb5d2..6e09ced3 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -40,10 +40,12 @@ elif [[ "$1" == "stack" && "$2" == "pre-install" ]]; then fi fi elif [[ "$1" == "stack" && "$2" == "install" ]]; then - if [[ "$CEPHADM_DEPLOY" = "True" ]]; then + if [[ "$CEPHADM_DEPLOY" = "True" && "$REMOTE_CEPH" = "False" ]]; then # Perform installation of service source echo_summary "[cephadm] Installing ceph" install_ceph + elif [[ "$CEPHADM_DEPLOY" = "True" && "$REMOTE_CEPH" = "True" ]]; then + echo "[CEPHADM] Remote Ceph: Skipping install" else # FIXME(melwitt): This is a hack to get around a namespacing issue with # Paste and PasteDeploy. For stable/queens, we use the Pike UCA packages