diff --git a/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch index 997475096..c1b5b79fd 100644 --- a/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch +++ b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch @@ -1,26 +1,40 @@ -From 38a9c61d87efb1385c8cf372bf013465d52632a1 Mon Sep 17 00:00:00 2001 +From 099b906fe716e4606bbb405632e0a80503d4a708 Mon Sep 17 00:00:00 2001 From: Chris Friesen Date: Fri, 3 Sep 2021 18:05:15 -0400 Subject: [PATCH] kubeadm: create platform pods with zero CPU resources -We want to specify zero CPU resources when creating the manifests +This specifies zero CPU resources when creating the manifests for the static platform pods, as a workaround for the lack of separate resource tracking for platform resources. -We also specify zero CPU and Memory resources for the -coredns deployment. -manifests.go appears to be the main file for this, not -sure if the others are used but I changed them just in case. +This specifies zero CPU and Memory resources for the coredns +deployment. manifests.go is the main source file for this, +not sure if the coredns.yaml are used but they are updated to +be consistent. + +This specifies CPU limit of 1 for kube-apiserver pod so that it is +treated as a burstable QoS. This gives a boost of cgroup CPUShares +since the burstable cgroup parent has significantly more CPUShares +than best-effort on typical systems. This improves kube-apiserver +API responsiveness. + +This increases kube-apiserver Readiness probe periodSeconds to 10 +based on WRS/SS joint recommendation for minimum probe settings. +This reduces likelihood of kube-apiserver probe failure and +subsequent pod-restart under servere load. This also reduces CPU +demand. Signed-off-by: Daniel Safta Signed-off-by: Boovan Rajendran +Signed-off-by: Jim Gauld --- - cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- - cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- - cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- - cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- - cmd/kubeadm/app/phases/controlplane/manifests.go | 6 +++--- - 5 files changed, 11 insertions(+), 11 deletions(-) + cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- + cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- + cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- + cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- + .../app/phases/controlplane/manifests.go | 8 +++++--- + cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++- + 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base index dea3749f217..908fd2b246a 100644 @@ -83,19 +97,21 @@ index 97c7f8b3e60..87df378cfc0 100644 volumeMounts: - name: config-volume diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go -index 8cd0bf577cd..b024263d8d1 100644 +index 8cd0bf577cd..03e5739f5dc 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go -@@ -63,7 +63,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -63,7 +63,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS), ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), - Resources: staticpodutil.ComponentResources("250m"), -+ Resources: staticpodutil.ComponentResources("0"), ++ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness; ++ // achieved by setting CPU Limits to make it burstable QoS. ++ Resources: staticpodutil.ComponentLimitResources("0", "1"), Env: kubeadmutil.GetProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer), map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}), -@@ -75,7 +75,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -75,7 +77,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), @@ -104,7 +120,7 @@ index 8cd0bf577cd..b024263d8d1 100644 Env: kubeadmutil.GetProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil), kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ -@@ -86,7 +86,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -86,7 +88,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), @@ -113,6 +129,41 @@ index 8cd0bf577cd..b024263d8d1 100644 Env: kubeadmutil.GetProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil), } +diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go +index 75efc4f0f98..46fa7a006ba 100644 +--- a/cmd/kubeadm/app/util/staticpod/utils.go ++++ b/cmd/kubeadm/app/util/staticpod/utils.go +@@ -92,6 +92,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements { + } + } + ++// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits ++func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements { ++ return v1.ResourceRequirements{ ++ Requests: v1.ResourceList{ ++ v1.ResourceCPU: resource.MustParse(cpu), ++ }, ++ Limits: v1.ResourceList{ ++ v1.ResourceCPU: resource.MustParse(lcpu), ++ }, ++ } ++} ++ + // NewVolume creates a v1.Volume with a hostPath mount to the specified location + func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume { + return v1.Volume{ +@@ -252,7 +264,10 @@ func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe { + func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe { + // sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks + // looking for "ready" status on kubeadm static Pods +- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1) ++ // WRS/SS joint recommendation: All pods probes should have following minimum probe ++ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10, ++ // timeoutSeconds 5, successThreshold 1, failureThreshold 3) ++ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10) + } + + // StartupProbe creates a Probe object with a HTTPGet handler -- 2.25.1 diff --git a/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch b/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch index 449e3dea8..b19907687 100644 --- a/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch +++ b/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch @@ -1,26 +1,40 @@ -From 1b5417b7c443c94ebce52df7d4b373242ba800c6 Mon Sep 17 00:00:00 2001 +From 0dc5e95fb37f7df6a3edb7cfed312554fcca8270 Mon Sep 17 00:00:00 2001 From: Chris Friesen Date: Fri, 3 Sep 2021 18:05:15 -0400 Subject: [PATCH] kubeadm: create platform pods with zero CPU resources -We want to specify zero CPU resources when creating the manifests +This specifies zero CPU resources when creating the manifests for the static platform pods, as a workaround for the lack of separate resource tracking for platform resources. -We also specify zero CPU and Memory resources for the -coredns deployment. -manifests.go appears to be the main file for this, not -sure if the others are used but I changed them just in case. +This specifies zero CPU and Memory resources for the coredns +deployment. manifests.go is the main source file for this, +not sure if the coredns.yaml are used but they are updated to +be consistent. + +This specifies CPU limit of 1 for kube-apiserver pod so that it is +treated as a burstable QoS. This gives a boost of cgroup CPUShares +since the burstable cgroup parent has significantly more CPUShares +than best-effort on typical systems. This improves kube-apiserver +API responsiveness. + +This increases kube-apiserver Readiness probe periodSeconds to 10 +based on WRS/SS joint recommendation for minimum probe settings. +This reduces likelihood of kube-apiserver probe failure and +subsequent pod-restart under servere load. This also reduces CPU +demand. Signed-off-by: Daniel Safta Signed-off-by: Boovan Rajendran +Signed-off-by: Jim Gauld --- - cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- - cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- - cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- - cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- - cmd/kubeadm/app/phases/controlplane/manifests.go | 6 +++--- - 5 files changed, 11 insertions(+), 11 deletions(-) + cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- + cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- + cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- + cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- + .../app/phases/controlplane/manifests.go | 8 +++++--- + cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++- + 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base index e03559423e6..49e88afc976 100644 @@ -83,19 +97,21 @@ index bb5455a15ff..24b7112ae81 100644 volumeMounts: - name: config-volume diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go -index 8cd0bf577cd..b024263d8d1 100644 +index 8cd0bf577cd..03e5739f5dc 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go -@@ -63,7 +63,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -63,7 +63,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS), ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), - Resources: staticpodutil.ComponentResources("250m"), -+ Resources: staticpodutil.ComponentResources("0"), ++ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness; ++ // achieved by setting CPU Limits to make it burstable QoS. ++ Resources: staticpodutil.ComponentLimitResources("0", "1"), Env: kubeadmutil.GetProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer), map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}), -@@ -75,7 +75,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -75,7 +77,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), @@ -104,7 +120,7 @@ index 8cd0bf577cd..b024263d8d1 100644 Env: kubeadmutil.GetProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil), kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ -@@ -86,7 +86,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -86,7 +88,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), @@ -113,6 +129,41 @@ index 8cd0bf577cd..b024263d8d1 100644 Env: kubeadmutil.GetProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil), } +diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go +index 56a3f25b72a..4ea5e4635df 100644 +--- a/cmd/kubeadm/app/util/staticpod/utils.go ++++ b/cmd/kubeadm/app/util/staticpod/utils.go +@@ -92,6 +92,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements { + } + } + ++// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits ++func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements { ++ return v1.ResourceRequirements{ ++ Requests: v1.ResourceList{ ++ v1.ResourceCPU: resource.MustParse(cpu), ++ }, ++ Limits: v1.ResourceList{ ++ v1.ResourceCPU: resource.MustParse(lcpu), ++ }, ++ } ++} ++ + // NewVolume creates a v1.Volume with a hostPath mount to the specified location + func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume { + return v1.Volume{ +@@ -245,7 +257,10 @@ func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe { + func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe { + // sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks + // looking for "ready" status on kubeadm static Pods +- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1) ++ // WRS/SS joint recommendation: All pods probes should have following minimum probe ++ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10, ++ // timeoutSeconds 5, successThreshold 1, failureThreshold 3) ++ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10) + } + + // StartupProbe creates a Probe object with a HTTPGet handler -- 2.25.1 diff --git a/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch b/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch index fa55d0315..1af5b6fdc 100644 --- a/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch +++ b/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch @@ -1,26 +1,40 @@ -From 7113cf5c9133a9da26002b6f02cc3b84ba0b55b8 Mon Sep 17 00:00:00 2001 +From 9f3efbfff49e3df7cb95fd58df7f649c2e580e35 Mon Sep 17 00:00:00 2001 From: Chris Friesen Date: Fri, 3 Sep 2021 18:05:15 -0400 Subject: [PATCH] kubeadm: create platform pods with zero CPU resources -We want to specify zero CPU resources when creating the manifests +This specifies zero CPU resources when creating the manifests for the static platform pods, as a workaround for the lack of separate resource tracking for platform resources. -We also specify zero CPU and Memory resources for the -coredns deployment. -manifests.go appears to be the main file for this, not -sure if the others are used but I changed them just in case. +This specifies zero CPU and Memory resources for the coredns +deployment. manifests.go is the main source file for this, +not sure if the coredns.yaml are used but they are updated to +be consistent. + +This specifies CPU limit of 1 for kube-apiserver pod so that it is +treated as a burstable QoS. This gives a boost of cgroup CPUShares +since the burstable cgroup parent has significantly more CPUShares +than best-effort on typical systems. This improves kube-apiserver +API responsiveness. + +This increases kube-apiserver Readiness probe periodSeconds to 10 +based on WRS/SS joint recommendation for minimum probe settings. +This reduces likelihood of kube-apiserver probe failure and +subsequent pod-restart under servere load. This also reduces CPU +demand. Signed-off-by: Daniel Safta Signed-off-by: Boovan Rajendran +Signed-off-by: Jim Gauld --- - cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- - cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- - cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- - cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- - cmd/kubeadm/app/phases/controlplane/manifests.go | 6 +++--- - 5 files changed, 11 insertions(+), 11 deletions(-) + cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- + cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- + cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- + cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- + .../app/phases/controlplane/manifests.go | 8 +++++--- + cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++- + 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base index e03559423e6..49e88afc976 100644 @@ -83,19 +97,21 @@ index 0e3c6c98c29..0aa23679caa 100644 volumeMounts: - name: config-volume diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go -index 73f4fa56270..da52342a6f6 100644 +index 73f4fa56270..343a9011498 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go -@@ -63,7 +63,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -63,7 +63,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS), ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), - Resources: staticpodutil.ComponentResources("250m"), -+ Resources: staticpodutil.ComponentResources("0"), ++ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness; ++ // achieved by setting CPU Limits to make it burstable QoS. ++ Resources: staticpodutil.ComponentLimitResources("0", "1"), Env: kubeadmutil.GetProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer), map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}), -@@ -75,7 +75,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -75,7 +77,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), @@ -104,7 +120,7 @@ index 73f4fa56270..da52342a6f6 100644 Env: kubeadmutil.GetProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil), kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ -@@ -86,7 +86,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -86,7 +88,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), @@ -113,6 +129,41 @@ index 73f4fa56270..da52342a6f6 100644 Env: kubeadmutil.GetProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil), } +diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go +index 56a3f25b72a..4ea5e4635df 100644 +--- a/cmd/kubeadm/app/util/staticpod/utils.go ++++ b/cmd/kubeadm/app/util/staticpod/utils.go +@@ -92,6 +92,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements { + } + } + ++// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits ++func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements { ++ return v1.ResourceRequirements{ ++ Requests: v1.ResourceList{ ++ v1.ResourceCPU: resource.MustParse(cpu), ++ }, ++ Limits: v1.ResourceList{ ++ v1.ResourceCPU: resource.MustParse(lcpu), ++ }, ++ } ++} ++ + // NewVolume creates a v1.Volume with a hostPath mount to the specified location + func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume { + return v1.Volume{ +@@ -245,7 +257,10 @@ func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe { + func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe { + // sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks + // looking for "ready" status on kubeadm static Pods +- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1) ++ // WRS/SS joint recommendation: All pods probes should have following minimum probe ++ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10, ++ // timeoutSeconds 5, successThreshold 1, failureThreshold 3) ++ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10) + } + + // StartupProbe creates a Probe object with a HTTPGet handler -- 2.25.1 diff --git a/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch b/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch index 8bf92be51..779d11aaa 100644 --- a/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch +++ b/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch @@ -1,26 +1,40 @@ -From 3c9fe997ea55a6d0ead592f796c24b77ebb6f0c6 Mon Sep 17 00:00:00 2001 +From 04dfde7f71f18f2681b746347a98b099187d4817 Mon Sep 17 00:00:00 2001 From: Chris Friesen Date: Fri, 3 Sep 2021 18:05:15 -0400 Subject: [PATCH] kubeadm: create platform pods with zero CPU resources -We want to specify zero CPU resources when creating the manifests +This specifies zero CPU resources when creating the manifests for the static platform pods, as a workaround for the lack of separate resource tracking for platform resources. -We also specify zero CPU and Memory resources for the -coredns deployment. -manifests.go appears to be the main file for this, not -sure if the others are used but I changed them just in case. +This specifies zero CPU and Memory resources for the coredns +deployment. manifests.go is the main source file for this, +not sure if the coredns.yaml are used but they are updated to +be consistent. + +This specifies CPU limit of 1 for kube-apiserver pod so that it is +treated as a burstable QoS. This gives a boost of cgroup CPUShares +since the burstable cgroup parent has significantly more CPUShares +than best-effort on typical systems. This improves kube-apiserver +API responsiveness. + +This increases kube-apiserver Readiness probe periodSeconds to 10 +based on WRS/SS joint recommendation for minimum probe settings. +This reduces likelihood of kube-apiserver probe failure and +subsequent pod-restart under servere load. This also reduces CPU +demand. Signed-off-by: Daniel Safta Signed-off-by: Boovan Rajendran +Signed-off-by: Jim Gauld --- - cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- - cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- - cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- - cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- - cmd/kubeadm/app/phases/controlplane/manifests.go | 6 +++--- - 5 files changed, 11 insertions(+), 11 deletions(-) + cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- + cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- + cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- + cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- + .../app/phases/controlplane/manifests.go | 8 +++++--- + cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++- + 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base index 8b6b2ab999c..58bd12ce5f2 100644 @@ -83,19 +97,21 @@ index 0e3c6c98c29..0aa23679caa 100644 volumeMounts: - name: config-volume diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go -index 73f4fa56270..da52342a6f6 100644 +index 73f4fa56270..343a9011498 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go -@@ -63,7 +63,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -63,7 +63,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS), ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), - Resources: staticpodutil.ComponentResources("250m"), -+ Resources: staticpodutil.ComponentResources("0"), ++ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness; ++ // achieved by setting CPU Limits to make it burstable QoS. ++ Resources: staticpodutil.ComponentLimitResources("0", "1"), Env: kubeadmutil.GetProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer), map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}), -@@ -75,7 +75,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -75,7 +77,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), @@ -104,7 +120,7 @@ index 73f4fa56270..da52342a6f6 100644 Env: kubeadmutil.GetProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil), kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ -@@ -86,7 +86,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -86,7 +88,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), @@ -113,6 +129,41 @@ index 73f4fa56270..da52342a6f6 100644 Env: kubeadmutil.GetProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil), } +diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go +index 0ed80c97e6b..eeda999daf2 100644 +--- a/cmd/kubeadm/app/util/staticpod/utils.go ++++ b/cmd/kubeadm/app/util/staticpod/utils.go +@@ -98,6 +98,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements { + } + } + ++// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits ++func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements { ++ return v1.ResourceRequirements{ ++ Requests: v1.ResourceList{ ++ v1.ResourceCPU: resource.MustParse(cpu), ++ }, ++ Limits: v1.ResourceList{ ++ v1.ResourceCPU: resource.MustParse(lcpu), ++ }, ++ } ++} ++ + // NewVolume creates a v1.Volume with a hostPath mount to the specified location + func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume { + return v1.Volume{ +@@ -251,7 +263,10 @@ func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe { + func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe { + // sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks + // looking for "ready" status on kubeadm static Pods +- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1) ++ // WRS/SS joint recommendation: All pods probes should have following minimum probe ++ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10, ++ // timeoutSeconds 5, successThreshold 1, failureThreshold 3) ++ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10) + } + + // StartupProbe creates a Probe object with a HTTPGet handler -- 2.25.1 diff --git a/kubernetes/kubernetes-1.28.4/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch b/kubernetes/kubernetes-1.28.4/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch index 50a748033..66a48a4a5 100644 --- a/kubernetes/kubernetes-1.28.4/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch +++ b/kubernetes/kubernetes-1.28.4/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch @@ -1,27 +1,41 @@ -From a04cda2d8623678fad18f4326af7deb649527a1f Mon Sep 17 00:00:00 2001 +From 8f247610176a7984dbae718bdacdabdb8bbf6f4d Mon Sep 17 00:00:00 2001 From: Saba Touheed Mujawar Date: Tue, 28 Nov 2023 09:16:45 -0500 Subject: [PATCH] kubeadm: create platform pods with zero CPU resources -We want to specify zero CPU resources when creating the manifests +This specifies zero CPU resources when creating the manifests for the static platform pods, as a workaround for the lack of separate resource tracking for platform resources. -We also specify zero CPU and Memory resources for the -coredns deployment. -manifests.go appears to be the main file for this, not -sure if the others are used but I changed them just in case. +This specifies zero CPU and Memory resources for the coredns +deployment. manifests.go is the main source file for this, +not sure if the coredns.yaml are used but they are updated to +be consistent. + +This specifies CPU limit of 1 for kube-apiserver pod so that it is +treated as a burstable QoS. This gives a boost of cgroup CPUShares +since the burstable cgroup parent has significantly more CPUShares +than best-effort on typical systems. This improves kube-apiserver +API responsiveness. + +This increases kube-apiserver Readiness probe periodSeconds to 10 +based on WRS/SS joint recommendation for minimum probe settings. +This reduces likelihood of kube-apiserver probe failure and +subsequent pod-restart under servere load. This also reduces CPU +demand. Signed-off-by: Daniel Safta Signed-off-by: Saba Touheed Mujawar Signed-off-by: Boovan Rajendran +Signed-off-by: Jim Gauld --- - cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- - cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- - cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- - cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- - cmd/kubeadm/app/phases/controlplane/manifests.go | 6 +++--- - 5 files changed, 11 insertions(+), 11 deletions(-) + cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- + cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- + cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- + cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- + .../app/phases/controlplane/manifests.go | 8 +++++--- + cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++- + 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base index 69c0f456591..ae65353534e 100644 @@ -84,19 +98,21 @@ index 931897b16e2..5c2b3c0daac 100644 volumeMounts: - name: config-volume diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go -index baa8ab6a965..e2c469a6e2f 100644 +index baa8ab6a965..0e3a6b326af 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go -@@ -66,7 +66,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -66,7 +66,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS), ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), - Resources: staticpodutil.ComponentResources("250m"), -+ Resources: staticpodutil.ComponentResources("0"), ++ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness; ++ // achieved by setting CPU Limits to make it burstable QoS. ++ Resources: staticpodutil.ComponentLimitResources("0", "1"), Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.APIServer.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer), map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}), -@@ -78,7 +78,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -78,7 +80,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), @@ -105,7 +121,7 @@ index baa8ab6a965..e2c469a6e2f 100644 Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.ControllerManager.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil), kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ -@@ -89,7 +89,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -89,7 +91,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), @@ -114,6 +130,41 @@ index baa8ab6a965..e2c469a6e2f 100644 Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.Scheduler.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil), } +diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go +index 4f74e7e84ed..73b300b4c0f 100644 +--- a/cmd/kubeadm/app/util/staticpod/utils.go ++++ b/cmd/kubeadm/app/util/staticpod/utils.go +@@ -98,6 +98,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements { + } + } + ++// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits ++func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements { ++ return v1.ResourceRequirements{ ++ Requests: v1.ResourceList{ ++ v1.ResourceCPU: resource.MustParse(cpu), ++ }, ++ Limits: v1.ResourceList{ ++ v1.ResourceCPU: resource.MustParse(lcpu), ++ }, ++ } ++} ++ + // NewVolume creates a v1.Volume with a hostPath mount to the specified location + func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume { + return v1.Volume{ +@@ -251,7 +263,10 @@ func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe { + func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe { + // sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks + // looking for "ready" status on kubeadm static Pods +- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1) ++ // WRS/SS joint recommendation: All pods probes should have following minimum probe ++ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10, ++ // timeoutSeconds 5, successThreshold 1, failureThreshold 3) ++ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10) + } + + // StartupProbe creates a Probe object with a HTTPGet handler -- 2.25.1 diff --git a/kubernetes/kubernetes-1.29.2/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch b/kubernetes/kubernetes-1.29.2/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch index 2ae43c70b..012d4f3b3 100644 --- a/kubernetes/kubernetes-1.29.2/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch +++ b/kubernetes/kubernetes-1.29.2/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch @@ -1,27 +1,41 @@ -From 759c2ad28ee3a23c0a4098ec711aaeb716c23f18 Mon Sep 17 00:00:00 2001 +From 668dc57e7c06da9b29dd677648fdb198901332a1 Mon Sep 17 00:00:00 2001 From: Boovan Rajendran Date: Mon, 26 Feb 2024 04:40:48 -0500 Subject: [PATCH] kubeadm: create platform pods with zero CPU resources -We want to specify zero CPU resources when creating the manifests +This specifies zero CPU resources when creating the manifests for the static platform pods, as a workaround for the lack of separate resource tracking for platform resources. -We also specify zero CPU and Memory resources for the -coredns deployment. -manifests.go appears to be the main file for this, not -sure if the others are used but I changed them just in case. +This specifies zero CPU and Memory resources for the coredns +deployment. manifests.go is the main source file for this, +not sure if the coredns.yaml are used but they are updated to +be consistent. + +This specifies CPU limit of 1 for kube-apiserver pod so that it is +treated as a burstable QoS. This gives a boost of cgroup CPUShares +since the burstable cgroup parent has significantly more CPUShares +than best-effort on typical systems. This improves kube-apiserver +API responsiveness. + +This increases kube-apiserver Readiness probe periodSeconds to 10 +based on WRS/SS joint recommendation for minimum probe settings. +This reduces likelihood of kube-apiserver probe failure and +subsequent pod-restart under servere load. This also reduces CPU +demand. Signed-off-by: Daniel Safta Signed-off-by: Saba Touheed Mujawar Signed-off-by: Boovan Rajendran +Signed-off-by: Jim Gauld --- - cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- - cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- - cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- - cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- - cmd/kubeadm/app/phases/controlplane/manifests.go | 6 +++--- - 5 files changed, 11 insertions(+), 11 deletions(-) + cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- + cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- + cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- + cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- + .../app/phases/controlplane/manifests.go | 8 +++++--- + cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++- + 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base index dd4570adb65..3a0fd7adb72 100644 @@ -84,19 +98,21 @@ index 905a2e050e6..2a2212d5d37 100644 volumeMounts: - name: config-volume diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go -index 998ca2e3456..f0879d41983 100644 +index 998ca2e3456..7988f1fe918 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go -@@ -66,7 +66,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -66,7 +66,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", endpoint.BindPort, v1.URISchemeHTTPS), ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", endpoint.BindPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", endpoint.BindPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), - Resources: staticpodutil.ComponentResources("250m"), -+ Resources: staticpodutil.ComponentResources("0"), ++ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness; ++ // achieved by setting CPU Limits to make it burstable QoS. ++ Resources: staticpodutil.ComponentLimitResources("0", "1"), Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.APIServer.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer), map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}), -@@ -78,7 +78,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -78,7 +80,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), @@ -105,7 +121,7 @@ index 998ca2e3456..f0879d41983 100644 Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.ControllerManager.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil), kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ -@@ -89,7 +89,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap +@@ -89,7 +91,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), @@ -114,6 +130,41 @@ index 998ca2e3456..f0879d41983 100644 Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.Scheduler.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil), } +diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go +index ea2b13f4b16..6f9afebf348 100644 +--- a/cmd/kubeadm/app/util/staticpod/utils.go ++++ b/cmd/kubeadm/app/util/staticpod/utils.go +@@ -99,6 +99,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements { + } + } + ++// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits ++func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements { ++ return v1.ResourceRequirements{ ++ Requests: v1.ResourceList{ ++ v1.ResourceCPU: resource.MustParse(cpu), ++ }, ++ Limits: v1.ResourceList{ ++ v1.ResourceCPU: resource.MustParse(lcpu), ++ }, ++ } ++} ++ + // NewVolume creates a v1.Volume with a hostPath mount to the specified location + func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume { + return v1.Volume{ +@@ -255,7 +267,10 @@ func LivenessProbe(host, path string, port int32, scheme v1.URIScheme) *v1.Probe + func ReadinessProbe(host, path string, port int32, scheme v1.URIScheme) *v1.Probe { + // sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks + // looking for "ready" status on kubeadm static Pods +- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1) ++ // WRS/SS joint recommendation: All pods probes should have following minimum probe ++ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10, ++ // timeoutSeconds 5, successThreshold 1, failureThreshold 3) ++ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10) + } + + // StartupProbe creates a Probe object with a HTTPGet handler -- 2.25.1