From 426486ce05680cbaea711531ba72e89f9afb17ce Mon Sep 17 00:00:00 2001 From: Jim Gauld Date: Thu, 17 Feb 2022 15:57:06 -0500 Subject: [PATCH] Patch script for restarting kubelet and isolcpus_plugin services This provides an example patch script for restarting kubelet and isolcpus_plugin services. TESTING: - PASS: Verified restart of kubelet and isolcpus_plugin services with designer patch install and remove. Story: 2008760 Task: 44541 Signed-off-by: Jim Gauld Change-Id: Idf624051b7238f39c4238ad72a7d7ffe14395b8b --- centos_pkg_dirs | 1 + .../centos/EXAMPLE_KUBELET.spec | 26 +++++ .../EXAMPLE_KUBELET/centos/build_srpm.data | 3 + .../scripts/kubelet-restart-example | 110 ++++++++++++++++++ 4 files changed, 140 insertions(+) create mode 100644 patch-scripts/EXAMPLE_KUBELET/centos/EXAMPLE_KUBELET.spec create mode 100644 patch-scripts/EXAMPLE_KUBELET/centos/build_srpm.data create mode 100644 patch-scripts/EXAMPLE_KUBELET/scripts/kubelet-restart-example diff --git a/centos_pkg_dirs b/centos_pkg_dirs index 3df065fe..3b90b8b6 100644 --- a/centos_pkg_dirs +++ b/centos_pkg_dirs @@ -8,4 +8,5 @@ patch-scripts/EXAMPLE_MTCE patch-scripts/EXAMPLE_VIM patch-scripts/EXAMPLE_SYSINV patch-scripts/EXAMPLE_SERVICE +patch-scripts/EXAMPLE_KUBELET enable-dev-patch diff --git a/patch-scripts/EXAMPLE_KUBELET/centos/EXAMPLE_KUBELET.spec b/patch-scripts/EXAMPLE_KUBELET/centos/EXAMPLE_KUBELET.spec new file mode 100644 index 00000000..3aa9527b --- /dev/null +++ b/patch-scripts/EXAMPLE_KUBELET/centos/EXAMPLE_KUBELET.spec @@ -0,0 +1,26 @@ +Name: EXAMPLE_KUBELET +Summary: StarlingX In-Service kubelet Patch Script Example +Version: 1.0 +Release: %{tis_patch_ver}%{?_tis_dist} +License: Apache-2.0 +Group: base +Packager: Wind River +Source0: kubelet-restart-example + +%install + install -Dp -m 700 %{S:0} %{buildroot}%{_patch_scripts}/%{name} + +%description +%{summary} + +%files +%defattr(-,root,root,-) +%{_patch_scripts}/* + +%post +cp -f %{_patch_scripts}/%{name} %{_runtime_patch_scripts}/ +exit 0 + +%preun +cp -f %{_patch_scripts}/%{name} %{_runtime_patch_scripts}/ +exit 0 diff --git a/patch-scripts/EXAMPLE_KUBELET/centos/build_srpm.data b/patch-scripts/EXAMPLE_KUBELET/centos/build_srpm.data new file mode 100644 index 00000000..0a4d919c --- /dev/null +++ b/patch-scripts/EXAMPLE_KUBELET/centos/build_srpm.data @@ -0,0 +1,3 @@ +COPY_LIST="scripts/*" +TIS_PATCH_VER=1 + diff --git a/patch-scripts/EXAMPLE_KUBELET/scripts/kubelet-restart-example b/patch-scripts/EXAMPLE_KUBELET/scripts/kubelet-restart-example new file mode 100644 index 00000000..79630f8a --- /dev/null +++ b/patch-scripts/EXAMPLE_KUBELET/scripts/kubelet-restart-example @@ -0,0 +1,110 @@ +#!/bin/bash +# +# Copyright (c) 2022 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# This script provides an example in-service patching restart, +# triggering a restart of the patching daemons themselves +# + +# +# The patching subsystem provides a patch-functions bash source file +# with useful function and variable definitions. +# +. /etc/patching/patch-functions + +# +# We can now check to see what type of node we're on, if it's locked, etc, +# and act accordingly +# + +# +# Declare an overall script return code +# +declare -i GLOBAL_RC=$PATCH_STATUS_OK + +# kubelet doesn't run on storage nodes +if is_storage +then + exit $GLOBAL_RC +fi + +if [ ! -f $PATCH_FLAGDIR/kubelet.restarted ] +then + # Check to see if kubelet is running + systemctl status kubelet.service|grep -q "Active: active (running)" + if [ $? -eq 0 ] + then + # issue a systemd daemon-reload once the rpms have been installed + # and before the processes have been restarted. + systemctl daemon-reload + + # Ask pmon to stop kubelet and isolcpu_plugin so that it won't raise + # any alarms as we force a restart + loginfo "$0: pmond stopping kubelet" + OLDPID=`pgrep -f /usr/bin/kubelet` + pmon-stop kubelet + pmon-stop isolcpu_plugin + + # Wait up to 30 seconds for service stop and enter a systemd + # auto-restart state + let -i UNTIL=$SECONDS+30 + while [ $UNTIL -ge $SECONDS ] + do + # Check to see if the process has stopped and switched states + systemctl status kubelet.service | grep -q "Active: activating (auto-restart)" + if [ $? -eq 0 ] + then + # Now that we are waiting on systemd to restart. Tell pmon + # to start this back up and enter it's delayed monitoring + # state + loginfo "$0: pmond starting kubelet" + pmon-start kubelet + pmon-start isolcpu_plugin + break + fi + + # Check every second to catch this auto-restart state + sleep 1 + done + + let -i UNTIL=$SECONDS+15 + while [ $UNTIL -ge $SECONDS ] + do + # Check to make sure the service is running + systemctl status kubelet.service | grep -q "Active: active (running)" + if [ $? -eq 0 ] + then + # Verify that it's not still the old process + NEWPID=`pgrep -f /usr/bin/kubelet` + if [ $? -eq 0 -a "$OLDPID" != "$NEWPID" ] + then + touch $PATCH_FLAGDIR/kubelet.restarted + break + fi + fi + + # Still not running? Let's wait 5 seconds and check again + sleep 5 + done + + systemctl status kubelet.service|grep -q "Active: active (running)" + STATUS=$? + NEWPID=`pgrep -f /usr/bin/kubelet` + if [ $STATUS -ne 0 -o $? -ne 0 -o "$OLDPID" = "$NEWPID" ] + then + # Still not running new kubelet! Clear the flag and mark the RC as failed + loginfo "$0: Failed to restart kubelet" + rm -f $PATCH_FLAGDIR/kubelet.restarted + GLOBAL_RC=$PATCH_STATUS_FAILED + fi + fi +fi + +# +# Exit the script with the overall return code +# +exit $GLOBAL_RC