Modify kube-apiserver cpu resource limits and readiness probes
This refactors the existing Kubernetes patch
kubeadm-create-platform-pods-with-zero-CPU-resources.patch.
This now modifies kube-apiserver pod to be burstable, and to
have relaxed Readiness probe settings.
This specifies CPU limit of 1 for kube-apiserver pod so that it is
treated as a burstable QoS. This gives a boost of cgroup CPUShares
since the burstable cgroup parent has significantly more CPUShares
than best-effort on typical systems. This improves kube-apiserver
API responsiveness.
This increases kube-apiserver Readiness probe periodSeconds to 10
based on WRS/SS joint recommendation for minimum probe settings.
This reduces likelihood of kube-apiserver probe failure and
subsequent pod-restart under servere load. This also reduces CPU
demand.
Partial-Bug: 2084714
TEST PLAN:
- PASS: AIO-SX: Fresh install with each k8s version
v1.24.4, 1.25.3, 1.26.1, 1.27.5, 1.28.4, 1.29.2,
verify kube-apiserver pod is burstable QoS and
has Readiness probe periodSeconds 10
e.g.,
kubectl get pod -n kube-system kube-apisever-controller-0 -oyaml
sudo kube-cpusets
- PASS: AIO-SX: orchestrated K8S upgrade from 1.24.4 to 1.29.2
Change-Id: Ic327b6c176c3a98c16afd14e6bc001315d7cbdc0
Signed-off-by: Jim Gauld <James.Gauld@windriver.com>
This commit is contained in:
Jim Gauld
parent
896f2eb78f
commit
45d5e92488
@ -1,26 +1,40 @@
|
||||
From 38a9c61d87efb1385c8cf372bf013465d52632a1 Mon Sep 17 00:00:00 2001
|
||||
From 099b906fe716e4606bbb405632e0a80503d4a708 Mon Sep 17 00:00:00 2001
|
||||
From: Chris Friesen <chris.friesen@windriver.com>
|
||||
Date: Fri, 3 Sep 2021 18:05:15 -0400
|
||||
Subject: [PATCH] kubeadm: create platform pods with zero CPU resources
|
||||
|
||||
We want to specify zero CPU resources when creating the manifests
|
||||
This specifies zero CPU resources when creating the manifests
|
||||
for the static platform pods, as a workaround for the lack of
|
||||
separate resource tracking for platform resources.
|
||||
|
||||
We also specify zero CPU and Memory resources for the
|
||||
coredns deployment.
|
||||
manifests.go appears to be the main file for this, not
|
||||
sure if the others are used but I changed them just in case.
|
||||
This specifies zero CPU and Memory resources for the coredns
|
||||
deployment. manifests.go is the main source file for this,
|
||||
not sure if the coredns.yaml are used but they are updated to
|
||||
be consistent.
|
||||
|
||||
This specifies CPU limit of 1 for kube-apiserver pod so that it is
|
||||
treated as a burstable QoS. This gives a boost of cgroup CPUShares
|
||||
since the burstable cgroup parent has significantly more CPUShares
|
||||
than best-effort on typical systems. This improves kube-apiserver
|
||||
API responsiveness.
|
||||
|
||||
This increases kube-apiserver Readiness probe periodSeconds to 10
|
||||
based on WRS/SS joint recommendation for minimum probe settings.
|
||||
This reduces likelihood of kube-apiserver probe failure and
|
||||
subsequent pod-restart under servere load. This also reduces CPU
|
||||
demand.
|
||||
|
||||
Signed-off-by: Daniel Safta <daniel.safta@windriver.com>
|
||||
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
|
||||
Signed-off-by: Jim Gauld <James.Gauld@windriver.com>
|
||||
---
|
||||
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
|
||||
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
|
||||
cmd/kubeadm/app/phases/controlplane/manifests.go | 6 +++---
|
||||
5 files changed, 11 insertions(+), 11 deletions(-)
|
||||
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
|
||||
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
|
||||
.../app/phases/controlplane/manifests.go | 8 +++++---
|
||||
cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++-
|
||||
6 files changed, 29 insertions(+), 12 deletions(-)
|
||||
|
||||
diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base
|
||||
index dea3749f217..908fd2b246a 100644
|
||||
@ -83,19 +97,21 @@ index 97c7f8b3e60..87df378cfc0 100644
|
||||
volumeMounts:
|
||||
- name: config-volume
|
||||
diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
index 8cd0bf577cd..b024263d8d1 100644
|
||||
index 8cd0bf577cd..03e5739f5dc 100644
|
||||
--- a/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
+++ b/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
@@ -63,7 +63,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -63,7 +63,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS),
|
||||
ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
- Resources: staticpodutil.ComponentResources("250m"),
|
||||
+ Resources: staticpodutil.ComponentResources("0"),
|
||||
+ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness;
|
||||
+ // achieved by setting CPU Limits to make it burstable QoS.
|
||||
+ Resources: staticpodutil.ComponentLimitResources("0", "1"),
|
||||
Env: kubeadmutil.GetProxyEnvVars(),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer),
|
||||
map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}),
|
||||
@@ -75,7 +75,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -75,7 +77,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
@ -104,7 +120,7 @@ index 8cd0bf577cd..b024263d8d1 100644
|
||||
Env: kubeadmutil.GetProxyEnvVars(),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil),
|
||||
kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
|
||||
@@ -86,7 +86,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -86,7 +88,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
@ -113,6 +129,41 @@ index 8cd0bf577cd..b024263d8d1 100644
|
||||
Env: kubeadmutil.GetProxyEnvVars(),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil),
|
||||
}
|
||||
diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
index 75efc4f0f98..46fa7a006ba 100644
|
||||
--- a/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
+++ b/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
@@ -92,6 +92,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements {
|
||||
}
|
||||
}
|
||||
|
||||
+// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits
|
||||
+func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements {
|
||||
+ return v1.ResourceRequirements{
|
||||
+ Requests: v1.ResourceList{
|
||||
+ v1.ResourceCPU: resource.MustParse(cpu),
|
||||
+ },
|
||||
+ Limits: v1.ResourceList{
|
||||
+ v1.ResourceCPU: resource.MustParse(lcpu),
|
||||
+ },
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
// NewVolume creates a v1.Volume with a hostPath mount to the specified location
|
||||
func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume {
|
||||
return v1.Volume{
|
||||
@@ -252,7 +264,10 @@ func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
|
||||
func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
|
||||
// sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks
|
||||
// looking for "ready" status on kubeadm static Pods
|
||||
- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1)
|
||||
+ // WRS/SS joint recommendation: All pods probes should have following minimum probe
|
||||
+ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10,
|
||||
+ // timeoutSeconds 5, successThreshold 1, failureThreshold 3)
|
||||
+ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10)
|
||||
}
|
||||
|
||||
// StartupProbe creates a Probe object with a HTTPGet handler
|
||||
--
|
||||
2.25.1
|
||||
|
||||
|
||||
@ -1,26 +1,40 @@
|
||||
From 1b5417b7c443c94ebce52df7d4b373242ba800c6 Mon Sep 17 00:00:00 2001
|
||||
From 0dc5e95fb37f7df6a3edb7cfed312554fcca8270 Mon Sep 17 00:00:00 2001
|
||||
From: Chris Friesen <chris.friesen@windriver.com>
|
||||
Date: Fri, 3 Sep 2021 18:05:15 -0400
|
||||
Subject: [PATCH] kubeadm: create platform pods with zero CPU resources
|
||||
|
||||
We want to specify zero CPU resources when creating the manifests
|
||||
This specifies zero CPU resources when creating the manifests
|
||||
for the static platform pods, as a workaround for the lack of
|
||||
separate resource tracking for platform resources.
|
||||
|
||||
We also specify zero CPU and Memory resources for the
|
||||
coredns deployment.
|
||||
manifests.go appears to be the main file for this, not
|
||||
sure if the others are used but I changed them just in case.
|
||||
This specifies zero CPU and Memory resources for the coredns
|
||||
deployment. manifests.go is the main source file for this,
|
||||
not sure if the coredns.yaml are used but they are updated to
|
||||
be consistent.
|
||||
|
||||
This specifies CPU limit of 1 for kube-apiserver pod so that it is
|
||||
treated as a burstable QoS. This gives a boost of cgroup CPUShares
|
||||
since the burstable cgroup parent has significantly more CPUShares
|
||||
than best-effort on typical systems. This improves kube-apiserver
|
||||
API responsiveness.
|
||||
|
||||
This increases kube-apiserver Readiness probe periodSeconds to 10
|
||||
based on WRS/SS joint recommendation for minimum probe settings.
|
||||
This reduces likelihood of kube-apiserver probe failure and
|
||||
subsequent pod-restart under servere load. This also reduces CPU
|
||||
demand.
|
||||
|
||||
Signed-off-by: Daniel Safta <daniel.safta@windriver.com>
|
||||
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
|
||||
Signed-off-by: Jim Gauld <James.Gauld@windriver.com>
|
||||
---
|
||||
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
|
||||
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
|
||||
cmd/kubeadm/app/phases/controlplane/manifests.go | 6 +++---
|
||||
5 files changed, 11 insertions(+), 11 deletions(-)
|
||||
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
|
||||
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
|
||||
.../app/phases/controlplane/manifests.go | 8 +++++---
|
||||
cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++-
|
||||
6 files changed, 29 insertions(+), 12 deletions(-)
|
||||
|
||||
diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base
|
||||
index e03559423e6..49e88afc976 100644
|
||||
@ -83,19 +97,21 @@ index bb5455a15ff..24b7112ae81 100644
|
||||
volumeMounts:
|
||||
- name: config-volume
|
||||
diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
index 8cd0bf577cd..b024263d8d1 100644
|
||||
index 8cd0bf577cd..03e5739f5dc 100644
|
||||
--- a/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
+++ b/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
@@ -63,7 +63,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -63,7 +63,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS),
|
||||
ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
- Resources: staticpodutil.ComponentResources("250m"),
|
||||
+ Resources: staticpodutil.ComponentResources("0"),
|
||||
+ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness;
|
||||
+ // achieved by setting CPU Limits to make it burstable QoS.
|
||||
+ Resources: staticpodutil.ComponentLimitResources("0", "1"),
|
||||
Env: kubeadmutil.GetProxyEnvVars(),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer),
|
||||
map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}),
|
||||
@@ -75,7 +75,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -75,7 +77,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
@ -104,7 +120,7 @@ index 8cd0bf577cd..b024263d8d1 100644
|
||||
Env: kubeadmutil.GetProxyEnvVars(),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil),
|
||||
kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
|
||||
@@ -86,7 +86,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -86,7 +88,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
@ -113,6 +129,41 @@ index 8cd0bf577cd..b024263d8d1 100644
|
||||
Env: kubeadmutil.GetProxyEnvVars(),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil),
|
||||
}
|
||||
diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
index 56a3f25b72a..4ea5e4635df 100644
|
||||
--- a/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
+++ b/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
@@ -92,6 +92,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements {
|
||||
}
|
||||
}
|
||||
|
||||
+// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits
|
||||
+func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements {
|
||||
+ return v1.ResourceRequirements{
|
||||
+ Requests: v1.ResourceList{
|
||||
+ v1.ResourceCPU: resource.MustParse(cpu),
|
||||
+ },
|
||||
+ Limits: v1.ResourceList{
|
||||
+ v1.ResourceCPU: resource.MustParse(lcpu),
|
||||
+ },
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
// NewVolume creates a v1.Volume with a hostPath mount to the specified location
|
||||
func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume {
|
||||
return v1.Volume{
|
||||
@@ -245,7 +257,10 @@ func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
|
||||
func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
|
||||
// sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks
|
||||
// looking for "ready" status on kubeadm static Pods
|
||||
- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1)
|
||||
+ // WRS/SS joint recommendation: All pods probes should have following minimum probe
|
||||
+ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10,
|
||||
+ // timeoutSeconds 5, successThreshold 1, failureThreshold 3)
|
||||
+ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10)
|
||||
}
|
||||
|
||||
// StartupProbe creates a Probe object with a HTTPGet handler
|
||||
--
|
||||
2.25.1
|
||||
|
||||
|
||||
@ -1,26 +1,40 @@
|
||||
From 7113cf5c9133a9da26002b6f02cc3b84ba0b55b8 Mon Sep 17 00:00:00 2001
|
||||
From 9f3efbfff49e3df7cb95fd58df7f649c2e580e35 Mon Sep 17 00:00:00 2001
|
||||
From: Chris Friesen <chris.friesen@windriver.com>
|
||||
Date: Fri, 3 Sep 2021 18:05:15 -0400
|
||||
Subject: [PATCH] kubeadm: create platform pods with zero CPU resources
|
||||
|
||||
We want to specify zero CPU resources when creating the manifests
|
||||
This specifies zero CPU resources when creating the manifests
|
||||
for the static platform pods, as a workaround for the lack of
|
||||
separate resource tracking for platform resources.
|
||||
|
||||
We also specify zero CPU and Memory resources for the
|
||||
coredns deployment.
|
||||
manifests.go appears to be the main file for this, not
|
||||
sure if the others are used but I changed them just in case.
|
||||
This specifies zero CPU and Memory resources for the coredns
|
||||
deployment. manifests.go is the main source file for this,
|
||||
not sure if the coredns.yaml are used but they are updated to
|
||||
be consistent.
|
||||
|
||||
This specifies CPU limit of 1 for kube-apiserver pod so that it is
|
||||
treated as a burstable QoS. This gives a boost of cgroup CPUShares
|
||||
since the burstable cgroup parent has significantly more CPUShares
|
||||
than best-effort on typical systems. This improves kube-apiserver
|
||||
API responsiveness.
|
||||
|
||||
This increases kube-apiserver Readiness probe periodSeconds to 10
|
||||
based on WRS/SS joint recommendation for minimum probe settings.
|
||||
This reduces likelihood of kube-apiserver probe failure and
|
||||
subsequent pod-restart under servere load. This also reduces CPU
|
||||
demand.
|
||||
|
||||
Signed-off-by: Daniel Safta <daniel.safta@windriver.com>
|
||||
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
|
||||
Signed-off-by: Jim Gauld <James.Gauld@windriver.com>
|
||||
---
|
||||
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
|
||||
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
|
||||
cmd/kubeadm/app/phases/controlplane/manifests.go | 6 +++---
|
||||
5 files changed, 11 insertions(+), 11 deletions(-)
|
||||
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
|
||||
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
|
||||
.../app/phases/controlplane/manifests.go | 8 +++++---
|
||||
cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++-
|
||||
6 files changed, 29 insertions(+), 12 deletions(-)
|
||||
|
||||
diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base
|
||||
index e03559423e6..49e88afc976 100644
|
||||
@ -83,19 +97,21 @@ index 0e3c6c98c29..0aa23679caa 100644
|
||||
volumeMounts:
|
||||
- name: config-volume
|
||||
diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
index 73f4fa56270..da52342a6f6 100644
|
||||
index 73f4fa56270..343a9011498 100644
|
||||
--- a/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
+++ b/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
@@ -63,7 +63,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -63,7 +63,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS),
|
||||
ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
- Resources: staticpodutil.ComponentResources("250m"),
|
||||
+ Resources: staticpodutil.ComponentResources("0"),
|
||||
+ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness;
|
||||
+ // achieved by setting CPU Limits to make it burstable QoS.
|
||||
+ Resources: staticpodutil.ComponentLimitResources("0", "1"),
|
||||
Env: kubeadmutil.GetProxyEnvVars(),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer),
|
||||
map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}),
|
||||
@@ -75,7 +75,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -75,7 +77,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
@ -104,7 +120,7 @@ index 73f4fa56270..da52342a6f6 100644
|
||||
Env: kubeadmutil.GetProxyEnvVars(),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil),
|
||||
kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
|
||||
@@ -86,7 +86,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -86,7 +88,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
@ -113,6 +129,41 @@ index 73f4fa56270..da52342a6f6 100644
|
||||
Env: kubeadmutil.GetProxyEnvVars(),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil),
|
||||
}
|
||||
diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
index 56a3f25b72a..4ea5e4635df 100644
|
||||
--- a/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
+++ b/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
@@ -92,6 +92,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements {
|
||||
}
|
||||
}
|
||||
|
||||
+// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits
|
||||
+func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements {
|
||||
+ return v1.ResourceRequirements{
|
||||
+ Requests: v1.ResourceList{
|
||||
+ v1.ResourceCPU: resource.MustParse(cpu),
|
||||
+ },
|
||||
+ Limits: v1.ResourceList{
|
||||
+ v1.ResourceCPU: resource.MustParse(lcpu),
|
||||
+ },
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
// NewVolume creates a v1.Volume with a hostPath mount to the specified location
|
||||
func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume {
|
||||
return v1.Volume{
|
||||
@@ -245,7 +257,10 @@ func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
|
||||
func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
|
||||
// sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks
|
||||
// looking for "ready" status on kubeadm static Pods
|
||||
- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1)
|
||||
+ // WRS/SS joint recommendation: All pods probes should have following minimum probe
|
||||
+ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10,
|
||||
+ // timeoutSeconds 5, successThreshold 1, failureThreshold 3)
|
||||
+ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10)
|
||||
}
|
||||
|
||||
// StartupProbe creates a Probe object with a HTTPGet handler
|
||||
--
|
||||
2.25.1
|
||||
|
||||
|
||||
@ -1,26 +1,40 @@
|
||||
From 3c9fe997ea55a6d0ead592f796c24b77ebb6f0c6 Mon Sep 17 00:00:00 2001
|
||||
From 04dfde7f71f18f2681b746347a98b099187d4817 Mon Sep 17 00:00:00 2001
|
||||
From: Chris Friesen <chris.friesen@windriver.com>
|
||||
Date: Fri, 3 Sep 2021 18:05:15 -0400
|
||||
Subject: [PATCH] kubeadm: create platform pods with zero CPU resources
|
||||
|
||||
We want to specify zero CPU resources when creating the manifests
|
||||
This specifies zero CPU resources when creating the manifests
|
||||
for the static platform pods, as a workaround for the lack of
|
||||
separate resource tracking for platform resources.
|
||||
|
||||
We also specify zero CPU and Memory resources for the
|
||||
coredns deployment.
|
||||
manifests.go appears to be the main file for this, not
|
||||
sure if the others are used but I changed them just in case.
|
||||
This specifies zero CPU and Memory resources for the coredns
|
||||
deployment. manifests.go is the main source file for this,
|
||||
not sure if the coredns.yaml are used but they are updated to
|
||||
be consistent.
|
||||
|
||||
This specifies CPU limit of 1 for kube-apiserver pod so that it is
|
||||
treated as a burstable QoS. This gives a boost of cgroup CPUShares
|
||||
since the burstable cgroup parent has significantly more CPUShares
|
||||
than best-effort on typical systems. This improves kube-apiserver
|
||||
API responsiveness.
|
||||
|
||||
This increases kube-apiserver Readiness probe periodSeconds to 10
|
||||
based on WRS/SS joint recommendation for minimum probe settings.
|
||||
This reduces likelihood of kube-apiserver probe failure and
|
||||
subsequent pod-restart under servere load. This also reduces CPU
|
||||
demand.
|
||||
|
||||
Signed-off-by: Daniel Safta <daniel.safta@windriver.com>
|
||||
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
|
||||
Signed-off-by: Jim Gauld <James.Gauld@windriver.com>
|
||||
---
|
||||
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
|
||||
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
|
||||
cmd/kubeadm/app/phases/controlplane/manifests.go | 6 +++---
|
||||
5 files changed, 11 insertions(+), 11 deletions(-)
|
||||
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
|
||||
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
|
||||
.../app/phases/controlplane/manifests.go | 8 +++++---
|
||||
cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++-
|
||||
6 files changed, 29 insertions(+), 12 deletions(-)
|
||||
|
||||
diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base
|
||||
index 8b6b2ab999c..58bd12ce5f2 100644
|
||||
@ -83,19 +97,21 @@ index 0e3c6c98c29..0aa23679caa 100644
|
||||
volumeMounts:
|
||||
- name: config-volume
|
||||
diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
index 73f4fa56270..da52342a6f6 100644
|
||||
index 73f4fa56270..343a9011498 100644
|
||||
--- a/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
+++ b/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
@@ -63,7 +63,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -63,7 +63,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS),
|
||||
ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
- Resources: staticpodutil.ComponentResources("250m"),
|
||||
+ Resources: staticpodutil.ComponentResources("0"),
|
||||
+ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness;
|
||||
+ // achieved by setting CPU Limits to make it burstable QoS.
|
||||
+ Resources: staticpodutil.ComponentLimitResources("0", "1"),
|
||||
Env: kubeadmutil.GetProxyEnvVars(),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer),
|
||||
map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}),
|
||||
@@ -75,7 +75,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -75,7 +77,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
@ -104,7 +120,7 @@ index 73f4fa56270..da52342a6f6 100644
|
||||
Env: kubeadmutil.GetProxyEnvVars(),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil),
|
||||
kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
|
||||
@@ -86,7 +86,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -86,7 +88,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
@ -113,6 +129,41 @@ index 73f4fa56270..da52342a6f6 100644
|
||||
Env: kubeadmutil.GetProxyEnvVars(),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil),
|
||||
}
|
||||
diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
index 0ed80c97e6b..eeda999daf2 100644
|
||||
--- a/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
+++ b/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
@@ -98,6 +98,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements {
|
||||
}
|
||||
}
|
||||
|
||||
+// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits
|
||||
+func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements {
|
||||
+ return v1.ResourceRequirements{
|
||||
+ Requests: v1.ResourceList{
|
||||
+ v1.ResourceCPU: resource.MustParse(cpu),
|
||||
+ },
|
||||
+ Limits: v1.ResourceList{
|
||||
+ v1.ResourceCPU: resource.MustParse(lcpu),
|
||||
+ },
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
// NewVolume creates a v1.Volume with a hostPath mount to the specified location
|
||||
func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume {
|
||||
return v1.Volume{
|
||||
@@ -251,7 +263,10 @@ func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
|
||||
func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
|
||||
// sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks
|
||||
// looking for "ready" status on kubeadm static Pods
|
||||
- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1)
|
||||
+ // WRS/SS joint recommendation: All pods probes should have following minimum probe
|
||||
+ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10,
|
||||
+ // timeoutSeconds 5, successThreshold 1, failureThreshold 3)
|
||||
+ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10)
|
||||
}
|
||||
|
||||
// StartupProbe creates a Probe object with a HTTPGet handler
|
||||
--
|
||||
2.25.1
|
||||
|
||||
|
||||
@ -1,27 +1,41 @@
|
||||
From a04cda2d8623678fad18f4326af7deb649527a1f Mon Sep 17 00:00:00 2001
|
||||
From 8f247610176a7984dbae718bdacdabdb8bbf6f4d Mon Sep 17 00:00:00 2001
|
||||
From: Saba Touheed Mujawar <sabatouheed.mujawar@windriver.com>
|
||||
Date: Tue, 28 Nov 2023 09:16:45 -0500
|
||||
Subject: [PATCH] kubeadm: create platform pods with zero CPU resources
|
||||
|
||||
We want to specify zero CPU resources when creating the manifests
|
||||
This specifies zero CPU resources when creating the manifests
|
||||
for the static platform pods, as a workaround for the lack of
|
||||
separate resource tracking for platform resources.
|
||||
|
||||
We also specify zero CPU and Memory resources for the
|
||||
coredns deployment.
|
||||
manifests.go appears to be the main file for this, not
|
||||
sure if the others are used but I changed them just in case.
|
||||
This specifies zero CPU and Memory resources for the coredns
|
||||
deployment. manifests.go is the main source file for this,
|
||||
not sure if the coredns.yaml are used but they are updated to
|
||||
be consistent.
|
||||
|
||||
This specifies CPU limit of 1 for kube-apiserver pod so that it is
|
||||
treated as a burstable QoS. This gives a boost of cgroup CPUShares
|
||||
since the burstable cgroup parent has significantly more CPUShares
|
||||
than best-effort on typical systems. This improves kube-apiserver
|
||||
API responsiveness.
|
||||
|
||||
This increases kube-apiserver Readiness probe periodSeconds to 10
|
||||
based on WRS/SS joint recommendation for minimum probe settings.
|
||||
This reduces likelihood of kube-apiserver probe failure and
|
||||
subsequent pod-restart under servere load. This also reduces CPU
|
||||
demand.
|
||||
|
||||
Signed-off-by: Daniel Safta <daniel.safta@windriver.com>
|
||||
Signed-off-by: Saba Touheed Mujawar <sabatouheed.mujawar@windriver.com>
|
||||
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
|
||||
Signed-off-by: Jim Gauld <James.Gauld@windriver.com>
|
||||
---
|
||||
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
|
||||
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
|
||||
cmd/kubeadm/app/phases/controlplane/manifests.go | 6 +++---
|
||||
5 files changed, 11 insertions(+), 11 deletions(-)
|
||||
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
|
||||
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
|
||||
.../app/phases/controlplane/manifests.go | 8 +++++---
|
||||
cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++-
|
||||
6 files changed, 29 insertions(+), 12 deletions(-)
|
||||
|
||||
diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base
|
||||
index 69c0f456591..ae65353534e 100644
|
||||
@ -84,19 +98,21 @@ index 931897b16e2..5c2b3c0daac 100644
|
||||
volumeMounts:
|
||||
- name: config-volume
|
||||
diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
index baa8ab6a965..e2c469a6e2f 100644
|
||||
index baa8ab6a965..0e3a6b326af 100644
|
||||
--- a/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
+++ b/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
@@ -66,7 +66,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -66,7 +66,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS),
|
||||
ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
- Resources: staticpodutil.ComponentResources("250m"),
|
||||
+ Resources: staticpodutil.ComponentResources("0"),
|
||||
+ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness;
|
||||
+ // achieved by setting CPU Limits to make it burstable QoS.
|
||||
+ Resources: staticpodutil.ComponentLimitResources("0", "1"),
|
||||
Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.APIServer.ExtraEnvs),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer),
|
||||
map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}),
|
||||
@@ -78,7 +78,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -78,7 +80,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
@ -105,7 +121,7 @@ index baa8ab6a965..e2c469a6e2f 100644
|
||||
Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.ControllerManager.ExtraEnvs),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil),
|
||||
kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
|
||||
@@ -89,7 +89,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -89,7 +91,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
@ -114,6 +130,41 @@ index baa8ab6a965..e2c469a6e2f 100644
|
||||
Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.Scheduler.ExtraEnvs),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil),
|
||||
}
|
||||
diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
index 4f74e7e84ed..73b300b4c0f 100644
|
||||
--- a/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
+++ b/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
@@ -98,6 +98,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements {
|
||||
}
|
||||
}
|
||||
|
||||
+// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits
|
||||
+func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements {
|
||||
+ return v1.ResourceRequirements{
|
||||
+ Requests: v1.ResourceList{
|
||||
+ v1.ResourceCPU: resource.MustParse(cpu),
|
||||
+ },
|
||||
+ Limits: v1.ResourceList{
|
||||
+ v1.ResourceCPU: resource.MustParse(lcpu),
|
||||
+ },
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
// NewVolume creates a v1.Volume with a hostPath mount to the specified location
|
||||
func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume {
|
||||
return v1.Volume{
|
||||
@@ -251,7 +263,10 @@ func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
|
||||
func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
|
||||
// sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks
|
||||
// looking for "ready" status on kubeadm static Pods
|
||||
- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1)
|
||||
+ // WRS/SS joint recommendation: All pods probes should have following minimum probe
|
||||
+ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10,
|
||||
+ // timeoutSeconds 5, successThreshold 1, failureThreshold 3)
|
||||
+ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10)
|
||||
}
|
||||
|
||||
// StartupProbe creates a Probe object with a HTTPGet handler
|
||||
--
|
||||
2.25.1
|
||||
|
||||
|
||||
@ -1,27 +1,41 @@
|
||||
From 759c2ad28ee3a23c0a4098ec711aaeb716c23f18 Mon Sep 17 00:00:00 2001
|
||||
From 668dc57e7c06da9b29dd677648fdb198901332a1 Mon Sep 17 00:00:00 2001
|
||||
From: Boovan Rajendran <boovan.rajendran@windriver.com>
|
||||
Date: Mon, 26 Feb 2024 04:40:48 -0500
|
||||
Subject: [PATCH] kubeadm: create platform pods with zero CPU resources
|
||||
|
||||
We want to specify zero CPU resources when creating the manifests
|
||||
This specifies zero CPU resources when creating the manifests
|
||||
for the static platform pods, as a workaround for the lack of
|
||||
separate resource tracking for platform resources.
|
||||
|
||||
We also specify zero CPU and Memory resources for the
|
||||
coredns deployment.
|
||||
manifests.go appears to be the main file for this, not
|
||||
sure if the others are used but I changed them just in case.
|
||||
This specifies zero CPU and Memory resources for the coredns
|
||||
deployment. manifests.go is the main source file for this,
|
||||
not sure if the coredns.yaml are used but they are updated to
|
||||
be consistent.
|
||||
|
||||
This specifies CPU limit of 1 for kube-apiserver pod so that it is
|
||||
treated as a burstable QoS. This gives a boost of cgroup CPUShares
|
||||
since the burstable cgroup parent has significantly more CPUShares
|
||||
than best-effort on typical systems. This improves kube-apiserver
|
||||
API responsiveness.
|
||||
|
||||
This increases kube-apiserver Readiness probe periodSeconds to 10
|
||||
based on WRS/SS joint recommendation for minimum probe settings.
|
||||
This reduces likelihood of kube-apiserver probe failure and
|
||||
subsequent pod-restart under servere load. This also reduces CPU
|
||||
demand.
|
||||
|
||||
Signed-off-by: Daniel Safta <daniel.safta@windriver.com>
|
||||
Signed-off-by: Saba Touheed Mujawar <sabatouheed.mujawar@windriver.com>
|
||||
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
|
||||
Signed-off-by: Jim Gauld <James.Gauld@windriver.com>
|
||||
---
|
||||
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
|
||||
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
|
||||
cmd/kubeadm/app/phases/controlplane/manifests.go | 6 +++---
|
||||
5 files changed, 11 insertions(+), 11 deletions(-)
|
||||
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
|
||||
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
|
||||
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
|
||||
.../app/phases/controlplane/manifests.go | 8 +++++---
|
||||
cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++-
|
||||
6 files changed, 29 insertions(+), 12 deletions(-)
|
||||
|
||||
diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base
|
||||
index dd4570adb65..3a0fd7adb72 100644
|
||||
@ -84,19 +98,21 @@ index 905a2e050e6..2a2212d5d37 100644
|
||||
volumeMounts:
|
||||
- name: config-volume
|
||||
diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
index 998ca2e3456..f0879d41983 100644
|
||||
index 998ca2e3456..7988f1fe918 100644
|
||||
--- a/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
+++ b/cmd/kubeadm/app/phases/controlplane/manifests.go
|
||||
@@ -66,7 +66,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -66,7 +66,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", endpoint.BindPort, v1.URISchemeHTTPS),
|
||||
ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", endpoint.BindPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", endpoint.BindPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
- Resources: staticpodutil.ComponentResources("250m"),
|
||||
+ Resources: staticpodutil.ComponentResources("0"),
|
||||
+ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness;
|
||||
+ // achieved by setting CPU Limits to make it burstable QoS.
|
||||
+ Resources: staticpodutil.ComponentLimitResources("0", "1"),
|
||||
Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.APIServer.ExtraEnvs),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer),
|
||||
map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}),
|
||||
@@ -78,7 +78,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -78,7 +80,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
@ -105,7 +121,7 @@ index 998ca2e3456..f0879d41983 100644
|
||||
Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.ControllerManager.ExtraEnvs),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil),
|
||||
kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
|
||||
@@ -89,7 +89,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
@@ -89,7 +91,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
|
||||
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
|
||||
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS),
|
||||
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
|
||||
@ -114,6 +130,41 @@ index 998ca2e3456..f0879d41983 100644
|
||||
Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.Scheduler.ExtraEnvs),
|
||||
}, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil),
|
||||
}
|
||||
diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
index ea2b13f4b16..6f9afebf348 100644
|
||||
--- a/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
+++ b/cmd/kubeadm/app/util/staticpod/utils.go
|
||||
@@ -99,6 +99,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements {
|
||||
}
|
||||
}
|
||||
|
||||
+// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits
|
||||
+func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements {
|
||||
+ return v1.ResourceRequirements{
|
||||
+ Requests: v1.ResourceList{
|
||||
+ v1.ResourceCPU: resource.MustParse(cpu),
|
||||
+ },
|
||||
+ Limits: v1.ResourceList{
|
||||
+ v1.ResourceCPU: resource.MustParse(lcpu),
|
||||
+ },
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
// NewVolume creates a v1.Volume with a hostPath mount to the specified location
|
||||
func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume {
|
||||
return v1.Volume{
|
||||
@@ -255,7 +267,10 @@ func LivenessProbe(host, path string, port int32, scheme v1.URIScheme) *v1.Probe
|
||||
func ReadinessProbe(host, path string, port int32, scheme v1.URIScheme) *v1.Probe {
|
||||
// sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks
|
||||
// looking for "ready" status on kubeadm static Pods
|
||||
- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1)
|
||||
+ // WRS/SS joint recommendation: All pods probes should have following minimum probe
|
||||
+ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10,
|
||||
+ // timeoutSeconds 5, successThreshold 1, failureThreshold 3)
|
||||
+ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10)
|
||||
}
|
||||
|
||||
// StartupProbe creates a Probe object with a HTTPGet handler
|
||||
--
|
||||
2.25.1
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user