diff --git a/tools/fixes/force_delete_statefulset.sh b/tools/fixes/force_delete_statefulset.sh new file mode 100755 index 000000000..41ecb6031 --- /dev/null +++ b/tools/fixes/force_delete_statefulset.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -e + +# This script should be run manually to forcefully evict statefulset +# pods from a failed control node. Statefulset pods in kubernetes are not +# evicted automatically if a node goes down. Those pods would go to +# an 'unknown' status. + +INITIAL_WAIT=${INITIAL_WAIT:-300} +LOOP_WAIT=${LOOP_WAIT:-10} + +if [[ -z "${1}" ]]; then + echo "ERROR: Failed control plane node name is not provided" + echo "Please provide an argument with the name of the failed control plane node" + exit 1 +fi + +failed_cp_node=$1 + +node_present=$(kubectl get nodes -l control-plane=enabled | grep "${failed_cp_node}" || true) + +if [[ -z "${node_present}" ]]; then + echo "The node name provided is incorrect." + echo "Please provide the name of failed control plane node." + exit 1 +fi + +# The idea here is to find out the names of the statefulset pod that is running on the provided +# failed control plane node and invoke a force delete action. + +for ns in $(kubectl get ns | awk '{print $1}');do + for ss in $(kubectl get statefulset -n "${ns}" --ignore-not-found --no-headers | \ + awk '{print $1}');do + label=$(kubectl -n "${ns}" get statefulset "${ss}" --show-labels --no-headers | \ + awk '{print $5}') + pod_name=$(kubectl -n "${ns}" get pod -l "${label}" -o wide --no-headers | \ + grep "${failed_cp_node}" | awk '{print $1}') + if [[ -z "${pod_name}" ]]; then + continue + else + kubectl -n "${ns}" delete pod "${pod_name}" --grace-period=0 --force --ignore-not-found + fi + done +done + +# The Logic below is to wait for StatefulSet pods to be ready. +# During testing it took approx 20 mins for all the evicted +# statefulset pods to be ready. This time could be either less or +# more. We will wait for 5 mins before we start checking on +# pods to be ready. +echo "waiting for ${INITIAL_WAIT} seconds before starting to check on pod readiness." +echo "This is the initial wait time." +sleep ${INITIAL_WAIT} + +for attempt in {1..180};do + echo "waiting for pods to be Ready" + echo "Trying attempt $attempt" + + # Statefulset pods are sequentially numbered. For example: 'mariadb-server-0', 'mariadb-server-1'. + # We do not have more than 3 replicas for a statefulset pod. + ss_not_ready_count=$(kubectl get --all-namespaces pods -o wide --no-headers | \ + grep '\-0 \|\-1 \|\-2 ' | egrep "0/1|0/2|0/3|1/2|1/3|2/3" | wc -l) + + # We would want the count of this variable to be '0' to determine that all the + # statefulset pods are ready. + if [ "$ss_not_ready_count" == 0 ]; then + echo "All the statefulset are now ready" + exit 0 + else + sleep "${LOOP_WAIT}" + fi +done +echo "Statefulset pods are not ready in the specified time limit" +echo "Manual intervention is needed to recover the statefulset pods" +exit 1