From e8cbdd0f7bbfc9d4c8651a4b63b827fa8d3310d8 Mon Sep 17 00:00:00 2001 From: Hediberto C Silva Date: Mon, 1 Sep 2025 15:55:33 -0300 Subject: [PATCH] Preserving the current states of Ceph SM services The ceph-osd and ceph-mon SM services have been presenting a state mismatch in some network recovery scenarios, where they are expected to be disabled by the SM, but reports as enabled-active based on Ceph availability. In this change, the proposal is to save the current SM state in the /var/run/ceph/.sm-ceph-mon-state and /var/run/ceph/.sm-ceph-osd-state files based on Start and Stop actions, and return disabled only if the save state is Stopped. In other cases, the ceph-init-wrapper script will process the status as before. In the ceph-init-wrapper script, we are limiting the current state update to executions called by the SM based on the parent process. This prevent external script uses such as manual interventions and PMON calls from affecting the behavior of SM, which is only used in AIO-DX setups. In addition, the flag used by the ceph-storage-network script has been renamed to maintain a pattern across created flags. This change is part of a solution to avoid the scenario where there is no active controller after a network recovery. From a storage perpective, all services are responding accordinly to SM requests. Other solutions for SM, mtcCLient or other services are needed to be adressed in future investigations. Test Plan: PASS: Fresh install for AIO-SX and AIO-DX. PASS: In virtual environments, simulate switch failure shutting down all interfaces from both controllers at same time and checking if Ceph services states are not in mismatch state. PASS: On AIO-DX, execute host-swact operations sucessfully. PASS: On AIO-DX, simulate BMC shutdown for standby controller and checking Ceph services states are correctly after booting. PASS: On AIO-DX, simulate BMC shutdown for active controller and checking if the Uncontrolled Swact happens sucessfully and if the Ceph services are correctly after booting. PASS: On AIO-DX, simulate DOR scenario shutting down both controllers at same time. Check if Ceph services are correctly after booting. Closes-bug: 2122117 Change-Id: Iafc0e30a441b1975ccfb98c16c4b30a53383d83e Signed-off-by: Hediberto C Silva --- ceph/ceph/files/ceph-init-wrapper.sh | 99 ++++++++++++++++++++++++- ceph/ceph/files/ceph-storage-network.sh | 2 +- 2 files changed, 97 insertions(+), 4 deletions(-) diff --git a/ceph/ceph/files/ceph-init-wrapper.sh b/ceph/ceph/files/ceph-init-wrapper.sh index 427eb693a..7958f741a 100755 --- a/ceph/ceph/files/ceph-init-wrapper.sh +++ b/ceph/ceph/files/ceph-init-wrapper.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2019-2024 Wind River Systems, Inc. +# Copyright (c) 2019-2025 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -76,6 +76,14 @@ LOCK_CEPH_OSD_STATUS_FILE="$VOLATILE_PATH/.ceph_osd_service" MONITOR_STATUS_TIMEOUT=30 MAX_STATUS_TIMEOUT=120 +STATE_RUNNING="Running" +STATE_STOPPED="Stopped" + +SM_CEPH_MON_STATE_FILE="$VOLATILE_PATH/ceph/.sm-ceph-mon-state" +SM_CEPH_OSD_STATE_FILE="$VOLATILE_PATH/ceph/.sm-ceph-osd-state" +SM_CEPH_MON_CURRENT_STATE=$(cat ${SM_CEPH_MON_STATE_FILE} 2>/dev/null) +SM_CEPH_OSD_CURRENT_STATE=$(cat ${SM_CEPH_OSD_STATE_FILE} 2>/dev/null) + RC=0 # SM can only pass arguments through environment variable @@ -91,6 +99,63 @@ else IFS=" " read -r -a args <<< "$@" fi +is_ppid_sm() +{ + local ppid_name + ppid_name=$(cat /proc/${PPID}/comm) + if [[ $ppid_name == "sm" ]]; then + return 0 + fi + return 1 +} + +# Save service current state +save_state() +{ + local ppid_name + local state_file="$1" + local new_state="$2" + if is_ppid_sm; then + if [[ ${state_file} == "${SM_CEPH_MON_STATE_FILE}" ]]; then + if [[ ${new_state} == "${SM_CEPH_MON_CURRENT_STATE}" ]]; then + return + fi + SM_CEPH_MON_CURRENT_STATE=${new_state} + elif [[ ${state_file} == "${SM_CEPH_OSD_STATE_FILE}" ]]; then + if [[ ${new_state} == "${SM_CEPH_OSD_CURRENT_STATE}" ]]; then + return + fi + SM_CEPH_OSD_CURRENT_STATE=${new_state} + fi + + echo "${new_state}" > "${state_file}" + log INFO "Updating ${state_file} state to ${new_state}." + else + ppid_name=$(cat /proc/${PPID}/comm) + log WARN "Cannot save state to ${state_file}. Expected 'sm' ppid, found '${ppid_name}'." + fi +} + +# Sanity check for the SM_CEPH_MON_CURRENT_STATE variable +if is_ppid_sm && [ "${SM_CEPH_MON_CURRENT_STATE}" != "${STATE_RUNNING}" ]; then + timeout 5 sm-query service ceph-mon 2>/dev/null | grep -q enabled-active + if [ $? -eq 0 ]; then + save_state ${SM_CEPH_MON_STATE_FILE} ${STATE_RUNNING} + else + save_state ${SM_CEPH_MON_STATE_FILE} ${STATE_STOPPED} + fi +fi + +# Sanity check for the SM_CEPH_OSD_CURRENT_STATE variable +if is_ppid_sm && [ "${SM_CEPH_OSD_CURRENT_STATE}" != "${STATE_RUNNING}" ]; then + timeout 5 sm-query service ceph-osd 2>/dev/null | grep -q enabled-active + if [ $? -eq 0 ]; then + save_state ${SM_CEPH_OSD_STATE_FILE} ${STATE_RUNNING} + else + save_state ${SM_CEPH_OSD_STATE_FILE} ${STATE_STOPPED} + fi +fi + # Log Management # Adding PID and PPID informations log () { @@ -346,6 +411,13 @@ start () # Start the service with_service_lock "${service}" ${CEPH_SCRIPT} start ${service} + + if [ "$RC" -eq 0 ] && [[ "${service}" == *"mon"* ]]; then + save_state ${SM_CEPH_MON_STATE_FILE} ${STATE_RUNNING} + elif [ "$RC" -eq 0 ] && [[ "${service}" == *"osd"* ]]; then + save_state ${SM_CEPH_OSD_STATE_FILE} ${STATE_RUNNING} + fi + log INFO "Ceph START ${service} command finished." } @@ -368,6 +440,13 @@ stop () fi with_service_lock "${service}" ${CEPH_SCRIPT} ${cmd} ${service} + + if [ "$RC" -eq 0 ] && [[ "${service}" == *"mon"* ]]; then + save_state ${SM_CEPH_MON_STATE_FILE} ${STATE_STOPPED} + elif [ "$RC" -eq 0 ] && [[ "${service}" == *"osd"* ]]; then + save_state ${SM_CEPH_OSD_STATE_FILE} ${STATE_STOPPED} + fi + log INFO "Ceph ${cmd^^} ${service} command finished." } @@ -448,12 +527,26 @@ status () eval target="$target" [ -z "${target}" ] && target="mon osd" + log INFO "status ${target}" + if [ ! -f ${CEPH_FILE} ]; then - # Ceph is not running on this node, return success + log INFO "Ceph is not running on this node, returning success." exit 0 fi - log INFO "status ${target}"; + if is_ppid_sm; then + if [[ "${target}" == *"mon"* ]] && \ + [[ "${SM_CEPH_MON_CURRENT_STATE}" != "${STATE_RUNNING}" ]]; then + log INFO "Ceph Mon is masked as ${SM_CEPH_MON_CURRENT_STATE} state, returning exit 1." + exit 1 + fi + + if [[ "${target}" == *"osd"* ]] && \ + [[ "${SM_CEPH_OSD_CURRENT_STATE}" != "${STATE_RUNNING}" ]]; then + log INFO "Ceph OSD is masked as ${SM_CEPH_OSD_CURRENT_STATE} state, returning exit 1." + exit 1 + fi + fi if [[ "$system_type" == "All-in-one" ]] && [[ "$system_mode" != "simplex" ]] && [[ "$target" == "osd" ]]; then has_ceph_network_carrier diff --git a/ceph/ceph/files/ceph-storage-network.sh b/ceph/ceph/files/ceph-storage-network.sh index b4510ea0b..b525be6c9 100755 --- a/ceph/ceph/files/ceph-storage-network.sh +++ b/ceph/ceph/files/ceph-storage-network.sh @@ -46,7 +46,7 @@ save_state() } RETVAL=0 -STATE_FILE="/var/run/ceph/.storage-networking-state" +STATE_FILE="/var/run/ceph/.sm-storage-networking-state" STATE_RUNNING="Running" STATE_STOPPED="Stopped" CURRENT_STATE=$(cat ${STATE_FILE} 2>/dev/null)