From f3c18b0f79e3b145d378474b24d861926dd61a13 Mon Sep 17 00:00:00 2001
From: Daniel Safta <daniel.safta@windriver.com>
Date: Wed, 9 Mar 2022 06:36:13 -0500
Subject: [PATCH] Add k8s container cleanup

When executing a reboot/shutdown
k8s pods are not receiving the SIGTERM
signal which leads some of them to
unexpected behaviour such as generating
huge coredumps.

There is an upstream issue regarding this:
https://github.com/kubernetes/kubernetes/issues/107158
The problem seems to be systemd related
but this commit addresses the problem
with a workaround.

This commit introduces a new script that
will cleanup all the remaing pods and will
be run after kubelet is stopped.

The script is executed successfully when
kubelet stops and the pods are stopped
before the system shuts down.

Closes-bug: 1964111
Signed-off-by: Daniel Safta <daniel.safta@windriver.com>
Change-Id: Ia0376aa510dd0dc3983e16cd89840726c15d6c92
---
 .../centos/files/kubeadm.conf                  |  1 +
 .../centos/files/kubeadm.conf                  |  1 +
 .../centos/files/k8s-container-cleanup         | 18 ++++++++++++++++++
 .../centos/kubernetes-unversioned.spec         |  3 +++
 4 files changed, 23 insertions(+)
 create mode 100755 kubernetes/kubernetes-unversioned/centos/files/k8s-container-cleanup

diff --git a/kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf b/kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf
index da0611444..aaadf6b36 100644
--- a/kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf
+++ b/kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf
@@ -13,6 +13,7 @@ ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/k
 ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
 ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
 ExecStopPost=/bin/rm -f /var/run/kubelet.pid
+ExecStopPost=-/usr/local/sbin/k8s-container-cleanup
 Restart=always
 StartLimitInterval=0
 RestartSec=10
diff --git a/kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf b/kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf
index da0611444..aaadf6b36 100644
--- a/kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf
+++ b/kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf
@@ -13,6 +13,7 @@ ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/k
 ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
 ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
 ExecStopPost=/bin/rm -f /var/run/kubelet.pid
+ExecStopPost=-/usr/local/sbin/k8s-container-cleanup
 Restart=always
 StartLimitInterval=0
 RestartSec=10
diff --git a/kubernetes/kubernetes-unversioned/centos/files/k8s-container-cleanup b/kubernetes/kubernetes-unversioned/centos/files/k8s-container-cleanup
new file mode 100755
index 000000000..f8bbc6568
--- /dev/null
+++ b/kubernetes/kubernetes-unversioned/centos/files/k8s-container-cleanup
@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright (c) 2022 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# The script will run everytime after the kubelet service is stopped.
+#
+# It will detect any unfinished pod and will try to send them SIGTERM
+# within 5s. If it times out, SIGKILL will be sent.
+#
+
+state=$(timeout 10 systemctl is-system-running)
+
+if [ "$state" = "stopping" ]; then
+    crictl ps | cut -d ' ' -f 1  | tail -n +2 | xargs -I {} crictl stop --timeout 5 {}
+fi
+
+exit 0
diff --git a/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec b/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec
index f9944f10c..959fc80d5 100644
--- a/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec
+++ b/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec
@@ -43,6 +43,7 @@ Source4: upgrade_k8s_config.sh
 
 Source5: sanitize_kubelet_reserved_cpus.sh
 
+Source6: k8s-container-cleanup
 Patch1: kubelet-service-remove-docker-dependency.patch
 
 BuildArch: noarch
@@ -105,6 +106,7 @@ install -m 700 %{SOURCE4} %{buildroot}/%{local_sbindir}/upgrade_k8s_config.sh
 
 install -m 700 %{SOURCE5} %{buildroot}/%{local_sbindir}/sanitize_kubelet_reserved_cpus.sh
 
+install -m 755 %{SOURCE6} %{buildroot}%{local_sbindir}/k8s-container-cleanup
 # install service files
 install -v -d -m 0755 %{buildroot}%{_unitdir}
 install -v -m 0644 -t %{buildroot}%{_unitdir} contrib/init/systemd/kubelet.service
@@ -125,6 +127,7 @@ install -v -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system.conf.d %{SOU
 # the following are execution scripts
 %{local_sbindir}/upgrade_k8s_config.sh
 %{local_sbindir}/sanitize_kubelet_reserved_cpus.sh
+%{local_sbindir}/k8s-container-cleanup
 
 # the following are symlinks
 %{_bindir}/kubeadm