Add method to reconfigure kubelet at runtime

This adds two runtime methods to reconfigure kubelet:
platform::kubernetes::master::update_kubelet_params::runtime
- this updates the kubelet-config ConfigMap with new parameters

platform::kubernetes::update_kubelet_config::runtime
- on each node, 'kubeadm upgrade node phase kubelet-config'
  is used to regenerate the /var/lib/kubelet/config.yaml file,
  then kubelet is restarted.

Along with this new configuration update mechanism, new kubelet-config
values from puppet are formatted with update script, i.e.,
imageGCHighThresholdPercent: 79
imageGCLowThresholdPercent: 75
evictionHard:
  imagefs.available: 2Gi

These new settings reduces likelihood of Node-Pressure Eviction
that occurs essentially near 86% /var/lib/docker usage. The default
upstream default imageGCHighThresholdPercent 85 is too high,
especially with evictionHard imagefs.available default of 15%.

The new image garbage collection parameters are engineered below
the system global default 80% file-system threshold. This allows
kubelet imageGC to cleanup space prior to hitting /var/lib/docker
alarms.

The evictionHard imagefs.available is reduced to 2Gi,
from the previous setting 15% which translated to 4.5Gi.

TESTING:
PASS - manually fill /var/lib/docker to exceed imageGC and
       verify GC operates
PASS - AIO-DX fresh install gets updated kubelet config
PASS - AIO-DX apply/remove designer patch with updated kubelet config
PASS - 'system kube-config-kubelet' updates K8S nodes kubelet config
PASS - AIO-DX reinstall controller-1 has updated kubelet config
PASS - AIO-DX install new worker node gets updated kubelet config

Partial-Bug: 1977754

Signed-off-by: Jim Gauld <james.gauld@windriver.com>
Change-Id: If634a8f59be3c13bf48612c7c67ca2802a03fc28

puppet-manifests/src/modules/platform/files/update_kubelet-config.py

@@ -0,0 +1,46 @@
+#
+# Copyright (c) 2022 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This script updates kubernetes evictionHard and imageGC parameters
+# to a yaml file containing kubernetes kubelet-config configmap.
+# The original input file is overwritten with merged values.
+
+import argparse
+import ruamel.yaml as yaml
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--configmap_file', required=True)
+parser.add_argument('--image_gc_low_threshold_percent', type=int, default=75)
+parser.add_argument('--image_gc_high_threshold_percent', type=int, default=79)
+parser.add_argument('--eviction_hard_imagefs_available', default='2Gi')
+args = parser.parse_args()
+
+configmap_file = args.configmap_file
+
+with open(configmap_file, 'r') as dest:
+    configmap = yaml.load(dest, Loader=yaml.RoundTripLoader)
+
+    # kubelet config is a single string. We need to parse the string
+    # in order to modify it correctly.
+    kubelet_config = yaml.load(configmap['data']['kubelet'],
+                               Loader=yaml.RoundTripLoader)
+
+    # Update imageGC parameters
+    kubelet_config['imageGCLowThresholdPercent'] = args.image_gc_low_threshold_percent
+    kubelet_config['imageGCHighThresholdPercent'] = args.image_gc_high_threshold_percent
+    kubelet_config.setdefault('evictionHard', {})
+    kubelet_config['evictionHard']['imagefs.available'] = args.eviction_hard_imagefs_available
+
+    kubelet_config_string = yaml.dump(kubelet_config, Dumper=yaml.RoundTripDumper,
+                                      default_flow_style=False)
+
+# use yaml.scalarstring.PreservedScalarString to make sure the yaml is
+# constructed with proper formatting and tabbing
+kubelet_config_string = yaml.scalarstring.PreservedScalarString(
+    kubelet_config_string)
+configmap['data']['kubelet'] = kubelet_config_string
+with open(configmap_file, 'w') as dest:
+    yaml.dump(configmap, dest, Dumper=yaml.RoundTripDumper,
+              default_flow_style=False)

puppet-manifests/src/modules/platform/manifests/kubernetes.pp

@@ -50,6 +50,9 @@ class platform::kubernetes::params (
   # The file holding the root CA cert/key to update to
   $rootca_certfile_new = '/etc/kubernetes/pki/ca_new.crt',
   $rootca_keyfile_new = '/etc/kubernetes/pki/ca_new.key',
+  $kubelet_image_gc_low_threshold_percent = 75,
+  $kubelet_image_gc_high_threshold_percent = 79,
+  $kubelet_eviction_hard_imagefs_available = '2Gi',
 ) { }
 
 class platform::kubernetes::configuration {
@@ -1397,3 +1400,43 @@ class platform::kubernetes::master::apiserver::runtime{
     command => "/usr/bin/kill -s SIGHUP $(pidof kube-apiserver)",
   }
 }
+
+class platform::kubernetes::master::update_kubelet_params::runtime
+  inherits ::platform::kubernetes::params {
+
+  # Update kubeadm bindmount if needed.
+  require platform::kubernetes::bindmounts
+
+  $kubelet_image_gc_low_threshold_percent = $::platform::kubernetes::params::kubelet_image_gc_low_threshold_percent
+  $kubelet_image_gc_high_threshold_percent = $::platform::kubernetes::params::kubelet_image_gc_high_threshold_percent
+  $kubelet_eviction_hard_imagefs_available = $::platform::kubernetes::params::kubelet_eviction_hard_imagefs_available
+
+  # Update kubelet parameters in kubelet-config Configmap.
+  exec { 'update kubelet config parameters':
+    environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf' ],
+    provider    => shell,
+    command     => template('platform/kube-config-kubelet.erb'),
+    timeout     => 60,
+    logoutput   => true,
+  }
+}
+
+class platform::kubernetes::update_kubelet_config::runtime
+  inherits ::platform::kubernetes::params {
+
+  # Update kubeadm/kubelet bindmounts if needed.
+  include platform::kubernetes::bindmounts
+
+  # Regenerate /var/lib/kubelet/config.yaml based on current kubelet-config
+  # ConfigMap. This does not regenerate /var/lib/kubelet/kubeadm-flags.env.
+  exec { 'update kubelet config':
+    environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf:/etc/kubernetes/kubelet.conf' ],
+    provider    => shell,
+    command     => 'kubeadm upgrade node phase kubelet-config',
+    timeout     => 60,
+    logoutput   => true,
+  }
+  -> exec { 'restart kubelet':
+      command => '/usr/local/sbin/pmon-restart kubelet'
+  }
+}

puppet-manifests/src/modules/platform/templates/kube-config-kubelet.erb

@@ -0,0 +1,61 @@
+<%# kubeadm stores cluster configuration and kubelet-config -%>
+<%# configuration as a configmaps in the cluster. This procedure -%>
+<%# keeps the configmap consistent and keeps kubelet managed by -%>
+<%# kubeadm. -%>
+
+<%# The kubelet-config configmap will be patched with updated kubelet -%>
+<%# parameters provided by the script update_kubelet-config.py. -%>
+
+s_exit() {
+    rm -v -f ${cm_kubelet_tempfile}
+    exit "${1:-0}"
+}
+
+# Temporary configuration file
+cm_kubelet_tempfile=$(mktemp)
+
+<% if @is_controller_active.to_s == 'true' or @system_mode == 'simplex' -%>
+# Get current kubelet-config versioned configmap name
+cm_name=$(kubectl -n kube-system get configmaps -oname 2>/dev/null | \
+          awk '/kubelet-config/ {print $1}')
+if [ $? -ne 0 ]; then
+    echo "Get configmaps failed."
+    s_exit 1
+fi
+echo "Got configmap: ${cm_name}"
+
+# Get pre-patched kubelet-config configmap
+kubectl -n kube-system get ${cm_name} -oyaml 2>/dev/null > ${cm_kubelet_tempfile}
+if [ $? -ne 0 ]; then
+    echo "Get ${cm_name} failed."
+    s_exit 1
+fi
+
+# Read and overwrite the kubelet-config YAML file with updated values.
+python /usr/share/puppet/modules/platform/files/update_kubelet-config.py \
+--configmap_file ${cm_kubelet_tempfile} \
+<%- if @kubelet_image_gc_low_threshold_percent -%>
+--image_gc_low_threshold_percent <%= @kubelet_image_gc_low_threshold_percent %> \
+<%- end -%>
+<%- if @kubelet_image_gc_high_threshold_percent -%>
+--image_gc_high_threshold_percent <%= @kubelet_image_gc_high_threshold_percent %> \
+<%- end -%>
+<%- if @kubelet_eviction_hard_imagefs_available -%>
+--eviction_hard_imagefs_available <%= @kubelet_eviction_hard_imagefs_available %>
+<%- end -%>
+
+if [ $? -ne 0 ]; then
+    echo "Update kubelet-config tempfile failed."
+    s_exit 1
+fi
+
+# Patch kubelet-config configmap with updated values.
+kubectl -n kube-system patch ${cm_name} -p "$(cat ${cm_kubelet_tempfile})"
+if [ $? -ne 0 ]; then
+    echo "Patch ${cm_name} failed."
+    s_exit 1
+fi
+<% end -%>
+
+# Success path exit and cleanup
+s_exit