update kubeadm configmap with new feature gate

The script will run for every k8s upgrade as a part of the
control-plane upgrade of the first master.
It updates kubeadm-config configmap to configure kube-apiserver
manifest with RemoveSelfLink=false.

The kubelet config override file is consumed by
kubeadm upgrade apply to configure cgroupDriver=cgroupfs.

Kubernetes changed default cgroupDriver cgroupfs to systemd
from k8s 1.21. We need to configure cgroupDriver with 'cgroupfs'
in the kubelet config otherwise kubelet will fail to launch.

Kubernetes changed kube-apiserver feature-gate RemoveSelfLink
default value to true from k8s 1.20 onwards.
This caused PVCs to not bound to PVs and eventually caused some
system apps to fail. We need to configure the kube-apiserver
feature-gates with RemoveSelfLink=false to avoid it until we can
update the application code to handle the new behaviour.

Test Plan:
PASS: successful testing of the script during k8s version upgrades
      from 1.18 -> 1.19 -> 1.20 -> 1.21 on AIO-DX system.
      Verified that PVCs are bound to PVs after each upgrade.
      Verified that pods are up and running after each upgrade.
      Verified that kube-apiserver manifest and kubelet config
      file is configured as expected after each upgrade.

PASS: 'build-pkgs --dep-test kubernetes-unversioned'
      package built with dependencies passed

Story: 2008972
Task: 44037

Signed-off-by: kdhokte <kaustubh.dhokte@windriver.com>
Change-Id: Ie53ebb5839eb0bd843f5bb2c577a4292969c0997

kubernetes/kubernetes-unversioned/centos/files/kubelet_override.yaml

@@ -0,0 +1,4 @@
+---
+kind: KubeletConfiguration
+apiVersion: kubelet.config.k8s.io/v1beta1
+cgroupDriver: cgroupfs

kubernetes/kubernetes-unversioned/centos/files/upgrade_k8s_config.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+# Copyright (c) 2021 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# This will run for every k8s upgrade as a part of the control-plane upgrade of the first master.
+# - updates kubeadm-config configmap to configure kube-apiserver manifest with RemoveSelfLink=false.
+# - generates a kubelet config override file to configure cgroupDriver=cgroupfs.
+#   This is consumed by kubeadm upgrade apply
+#
+# Background:
+# Kubernetes 1.21 changed cgroupDriver default to systemd (was cgroupfs).
+# Kubernetes 1.20 changed feature-gates RemoveSelfLink default to true.
+
+
+KUBEADM_CONFIGMAP_TMPFILE='/tmp/kubeadm_cm.yaml'
+
+function log {
+    logger -p local1.info "$1"
+}
+
+# Update the configmap for kubeadm
+function update_apiserver_configmap {
+
+    log "Retrieving kubeadm configmap to temporary location: ${KUBEADM_CONFIGMAP_TMPFILE}"
+    counter=0
+    RC=0
+    RETRIES=10
+    until  [ $counter -gt $RETRIES ]; do
+        kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system get \
+            configmap kubeadm-config -o yaml > ${KUBEADM_CONFIGMAP_TMPFILE}
+        RC=$?
+        if [ "$RC" = "0" ] ; then
+            log "Kubeadm configmap retrieved."
+            break
+        fi
+        log "Error retrieving kubeadm configmap, retrying..."
+        sleep 5
+        let "counter+=1"
+    done
+
+    if [ $counter -gt $RETRIES ]; then
+        log "Failed to retrieve kubeadm configmap with error code [$RC]".
+        exit $RC
+    fi
+
+    if ! grep -q 'RemoveSelfLink=false' ${KUBEADM_CONFIGMAP_TMPFILE}; then
+
+        log "Updating kube-apiserver feature-gates in retrieved kubeadm-config"
+
+        if sed -i \
+'/^\s*feature-gates:\s*.*HugePageStorageMediumSize='\
+'true/ s/$/,RemoveSelfLink=false/' ${KUBEADM_CONFIGMAP_TMPFILE}; then
+
+            if grep -q 'RemoveSelfLink=false' ${KUBEADM_CONFIGMAP_TMPFILE};
+            then
+                log "Successfully updated retrieved kubeadm-config"
+                if kubectl --kubeconfig=/etc/kubernetes/admin.conf replace -f \
+                        ${KUBEADM_CONFIGMAP_TMPFILE}; then
+                    log 'Successfully replaced updated kubeadm configmap.'
+                else
+                    RC=$?
+                    log "Failed to replace updated kubeadm configmap with error code: [$RC]"
+                    exit $RC
+                fi
+            else
+                log 'Failed to update kube-apiserver feature-gates with an unknown error'
+                exit -1
+            fi
+        else
+            RC=$?
+            log "Failed to update ${KUBEADM_CONFIGMAP_TMPFILE} with error code: [$RC]"
+            exit $RC
+        fi
+    else
+        log "Kubeadm configmap was already updated with RemoveSelfLink=false. Nothing to do."
+    fi
+
+    rm -f ${KUBEADM_CONFIGMAP_TMPFILE}
+
+}
+
+update_apiserver_configmap
+exit 0

kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec

@@ -5,6 +5,7 @@
 #
 
 %define debug_package %{nil}
+%define local_sbindir /usr/local/sbin
 
 %global _k8s_name kubernetes
 
@@ -35,6 +36,11 @@ Source1: %{con_repo}-v%{con_commit}.tar.gz
 # systemd resource control enable CPU and Memory accounting for cgroups
 Source2: kubernetes-accounting.conf
 
+# kubelet config overrides parameters
+Source3: kubelet_override.yaml
+
+Source4: upgrade_k8s_config.sh
+
 Patch1: kubelet-service-remove-docker-dependency.patch
 
 BuildArch: noarch
@@ -89,6 +95,11 @@ install -v -d -m 0755 %{buildroot}%{_tmpfilesdir}
 install -v -p -m 0644 -t %{buildroot}/%{_tmpfilesdir} contrib/init/systemd/tmpfiles.d/kubernetes.conf
 mkdir -p %{buildroot}/run
 install -v -d -m 0755 %{buildroot}/run/%{_k8s_name}/
+install -p -D -m 644 %{SOURCE3} %{buildroot}%{_sysconfdir}/%{_k8s_name}/kubelet_override.yaml
+
+install -d %{buildroot}%{local_sbindir}
+# install execution scripts
+install -m 700 %{SOURCE4} %{buildroot}/%{local_sbindir}/upgrade_k8s_config.sh
 
 # install service files
 install -v -d -m 0755 %{buildroot}%{_unitdir}
@@ -107,6 +118,9 @@ install -v -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system.conf.d %{SOU
 %dir %{_curr_stage1}
 %dir %{_curr_stage2}
 
+# the following are execution scripts
+%{local_sbindir}/upgrade_k8s_config.sh
+
 # the following are symlinks
 %{_bindir}/kubeadm
 %{_bindir}/kubelet
@@ -123,6 +137,7 @@ install -v -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system.conf.d %{SOU
 %config(noreplace) %{_sysconfdir}/%{_k8s_name}/config
 %config(noreplace) %{_sysconfdir}/%{_k8s_name}/kubelet
 %config(noreplace) %{_sysconfdir}/%{_k8s_name}/kubelet.kubeconfig
+%config(noreplace) %{_sysconfdir}/%{_k8s_name}/kubelet_override.yaml
 %config(noreplace) %{_sysconfdir}/%{_k8s_name}/proxy
 %config(noreplace) %{_sysconfdir}/systemd/system.conf.d/kubernetes-accounting.conf
 %{_tmpfilesdir}/kubernetes.conf