Erickson Silva de Oliveira
790bcd1c9c
During an upgrade, an unexpected daemon crash may occur, and as a result, ceph may get the 'HEALTH_WARN' status caused by 'X daemon crashed recently'. This alarm occurs when a process is unexpectedly interrupted or a failure occurs. However, after restarting the component, everything returns to normal. However, the Ceph status remains HEATLH_WARN because the failure is considered recent. This causes the platform upgrade to fail at stages where alarms are not expected. To prevent this behavior during the upgrade, this recent crash alarm is disabled when 'software deploy start' is executed. At the end of the upgrade process, when 'software deploy delete' is executed, it is re-enabled. Test Plan: - PASS: Upgrade on AIO-SX with rook-ceph backend configured. - PASS: Upgrade on AIO-SX with ceph backend configured. - PASS: Upgrade on STD with rook-ceph backend configured. - PASS: Rollback on AIO-SX with rook-ceph backend configured. Closes-Bug: 2127778 Change-Id: I9a95dfa24f4ce3ea07ea64f4e48da61027dc30b3 Signed-off-by: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
50 lines
1.6 KiB
Bash
50 lines
1.6 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Copyright (c) 2025 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
# When the rook ceph backend is configured, this script archives any daemon
|
|
# crashes that may have occurred during the upgrade, and restores the value
|
|
# of the mgr/crash/warn_recent_interval key that was modified at the
|
|
# beginning of the upgrade, at deploy-start.
|
|
#
|
|
|
|
# The script receives these parameters:
|
|
FROM_RELEASE=$1
|
|
TO_RELEASE=$2
|
|
ACTION=$3
|
|
|
|
SOFTWARE_LOG_PATH="/var/log/software.log"
|
|
ROOK_CEPH_CONFIGURED_FLAG="/etc/platform/.node_rook_configured"
|
|
|
|
# Default logging method extracted from script #02
|
|
function log {
|
|
echo "$(date -Iseconds | cut -d'+' -f1): ${NAME}[$$]: INFO: $*" \
|
|
>> "${SOFTWARE_LOG_PATH}" 2>&1
|
|
}
|
|
|
|
# Script start
|
|
if [[ "${ACTION}" != "delete" ]]; then
|
|
log "No actions required from ${FROM_RELEASE} to ${TO_RELEASE} with action ${ACTION}."
|
|
exit 0
|
|
fi
|
|
|
|
# Checks if rook ceph is configured
|
|
if [ -f "$ROOK_CEPH_CONFIGURED_FLAG" ]; then
|
|
# Archive rook ceph daemon crashes
|
|
log "Archiving rook ceph daemon crashes.."
|
|
ceph crash archive-all
|
|
# Restore warn_recent_interval value
|
|
warn_recent_interval=$(ceph config-key get usm/mgr/crash/warn_recent_interval 2>/dev/null)
|
|
if [ $? -ne 0 ] || [ -z "$warn_recent_interval" ]; then
|
|
warn_recent_interval="1209600"
|
|
fi
|
|
ceph config set mgr mgr/crash/warn_recent_interval "$warn_recent_interval"
|
|
ceph config-key del usm/mgr/crash/warn_recent_interval > /dev/null 2>&1
|
|
log "Successfully enabled rook ceph crash alarms."
|
|
else
|
|
log "Rook Ceph backend is not configured. Skipping."
|
|
fi
|
|
exit 0
|