140 lines
5.4 KiB
Diff
140 lines
5.4 KiB
Diff
From 66f4a61ce77624dcc0b520e01fccacec61dfea37 Mon Sep 17 00:00:00 2001
|
|
From: Jim Gauld <james.gauld@windriver.com>
|
|
Date: Tue, 7 Sep 2021 14:21:03 -0400
|
|
Subject: [PATCH 3/7] kubelet cpumanager infrastructure pods use system
|
|
reserved CPUs
|
|
|
|
This assigns system infrastructure pods to the "reserved" cpuset
|
|
to isolate them from the shared pool of CPUs.
|
|
|
|
Infrastructure pods include any pods that belong to the kube-system,
|
|
armada, cert-manager, vault, platform-deployment-manager, portieris,
|
|
notification or flux-helm namespaces.
|
|
|
|
The implementation is a bit simplistic, it is assumed that the
|
|
"reserved" cpuset is large enough to handle all infrastructure pods
|
|
CPU allocations.
|
|
|
|
This also prevents infrastucture pods from using Guaranteed resources.
|
|
|
|
Signed-off-by: Jim Gauld <james.gauld@windriver.com>
|
|
---
|
|
pkg/kubelet/cm/cpumanager/policy_static.go | 44 +++++++++++++++++++
|
|
.../cm/cpumanager/policy_static_test.go | 19 +++++++-
|
|
2 files changed, 62 insertions(+), 1 deletion(-)
|
|
|
|
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
|
|
index 94d81796132..b48a5b997a3 100644
|
|
--- a/pkg/kubelet/cm/cpumanager/policy_static.go
|
|
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go
|
|
@@ -32,6 +32,11 @@ import (
|
|
// PolicyStatic is the name of the static policy
|
|
const PolicyStatic policyName = "static"
|
|
|
|
+// Define namespaces used by platform infrastructure pods
|
|
+var infraNamespaces = [...]string{
|
|
+ "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm",
|
|
+}
|
|
+
|
|
// staticPolicy is a CPU manager policy that does not change CPU
|
|
// assignments for exclusively pinned guaranteed containers after the main
|
|
// container process starts.
|
|
@@ -232,6 +237,31 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c
|
|
}
|
|
|
|
func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) error {
|
|
+ // Process infra pods before guaranteed pods
|
|
+ if isKubeInfra(pod) {
|
|
+ // Container belongs in reserved pool.
|
|
+ // We don't want to fall through to the p.guaranteedCPUs() clause below so return either nil or error.
|
|
+ if _, ok := s.GetCPUSet(string(pod.UID), container.Name); ok {
|
|
+ klog.Infof("[cpumanager] static policy: reserved container already present in state, skipping " +
|
|
+ "(namespace: %s, pod UID: %s, pod: %s, container: %s)",
|
|
+ pod.Namespace, string(pod.UID), pod.Name, container.Name)
|
|
+ return nil
|
|
+ }
|
|
+
|
|
+ cpuset := p.reserved
|
|
+ if cpuset.IsEmpty() {
|
|
+ // If this happens then someone messed up.
|
|
+ return fmt.Errorf("[cpumanager] static policy: reserved container unable to allocate cpus " +
|
|
+ "(namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v, reserved:%v",
|
|
+ pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset, p.reserved)
|
|
+ }
|
|
+ s.SetCPUSet(string(pod.UID), container.Name, cpuset)
|
|
+ klog.Infof("[cpumanager] static policy: reserved: AddContainer " +
|
|
+ "(namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v",
|
|
+ pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset)
|
|
+ return nil
|
|
+ }
|
|
+
|
|
if numCPUs := p.guaranteedCPUs(pod, container); numCPUs != 0 {
|
|
klog.InfoS("Static policy: Allocate", "pod", klog.KObj(pod), "containerName", container.Name)
|
|
// container belongs in an exclusively allocated pool
|
|
@@ -321,6 +351,10 @@ func (p *staticPolicy) guaranteedCPUs(pod *v1.Pod, container *v1.Container) int
|
|
if cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() {
|
|
return 0
|
|
}
|
|
+ // Infrastructure pods use reserved CPUs even if they're in the Guaranteed QoS class
|
|
+ if isKubeInfra(pod) {
|
|
+ return 0
|
|
+ }
|
|
// Safe downcast to do for all systems with < 2.1 billion CPUs.
|
|
// Per the language spec, `int` is guaranteed to be at least 32 bits wide.
|
|
// https://golang.org/ref/spec#Numeric_types
|
|
@@ -523,3 +557,13 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
|
|
|
|
return hints
|
|
}
|
|
+
|
|
+// check if a given pod is in a platform infrastructure namespace
|
|
+func isKubeInfra(pod *v1.Pod) bool {
|
|
+ for _, namespace := range infraNamespaces {
|
|
+ if namespace == pod.Namespace {
|
|
+ return true
|
|
+ }
|
|
+ }
|
|
+ return false
|
|
+}
|
|
\ No newline at end of file
|
|
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
|
|
index 93a2870532a..90ad98e71bc 100644
|
|
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
|
|
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
|
|
@@ -747,7 +747,8 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
|
|
}
|
|
|
|
func TestStaticPolicyAddWithResvList(t *testing.T) {
|
|
-
|
|
+ infraPod := makePod("fakePod", "fakeContainer2", "200m", "200m")
|
|
+ infraPod.Namespace = "kube-system"
|
|
testCases := []staticPolicyTestWithResvList{
|
|
{
|
|
description: "GuPodSingleCore, SingleSocketHT, ExpectError",
|
|
@@ -789,6 +790,22 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
|
|
expCPUAlloc: true,
|
|
expCSet: cpuset.NewCPUSet(4, 5),
|
|
},
|
|
+ {
|
|
+ description: "InfraPod, SingleSocketHT, ExpectAllocReserved",
|
|
+ topo: topoSingleSocketHT,
|
|
+ numReservedCPUs: 2,
|
|
+ reserved: cpuset.NewCPUSet(0, 1),
|
|
+ stAssignments: state.ContainerCPUAssignments{
|
|
+ "fakePod": map[string]cpuset.CPUSet{
|
|
+ "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
|
|
+ },
|
|
+ },
|
|
+ stDefaultCPUSet: cpuset.NewCPUSet(4, 5),
|
|
+ pod: infraPod,
|
|
+ expErr: nil,
|
|
+ expCPUAlloc: true,
|
|
+ expCSet: cpuset.NewCPUSet(0, 1),
|
|
+ },
|
|
}
|
|
|
|
testExcl := true
|
|
--
|
|
2.17.1
|
|
|