From 777b7d88630bae55bf130e240212a2abf288bbd3 Mon Sep 17 00:00:00 2001 From: Chris Friesen Date: Mon, 26 Oct 2020 17:30:00 -0400 Subject: [PATCH] enable support for kubernetes to ignore isolcpus The normal mechanisms for allocating isolated CPUs do not allow a mix of isolated and exclusive CPUs in the same container. In order to allow this in *very* limited cases where the pod spec is known in advance we will add the ability to disable the normal isolcpus behaviour. If the file "/etc/kubernetes/ignore_isolcpus" exists, then kubelet will basically forget everything it knows about isolcpus and just treat them like regular CPUs. The admin user can then rely on the fact that CPU allocation is deterministic to ensure that the isolcpus they configure end up being allocated to the correct pods. Story: 2008760 Task: 42164 Change-Id: Ie38c81209ee407ac98b4882f2581fc14622b3af1 Signed-off-by: Chris Friesen --- ...-option-to-disable-isolcpu-awareness.patch | 79 +++++++++++++++++++ kubernetes/kubernetes/centos/kubernetes.spec | 2 + 2 files changed, 81 insertions(+) create mode 100644 kubernetes/kubernetes/centos/files/add-option-to-disable-isolcpu-awareness.patch diff --git a/kubernetes/kubernetes/centos/files/add-option-to-disable-isolcpu-awareness.patch b/kubernetes/kubernetes/centos/files/add-option-to-disable-isolcpu-awareness.patch new file mode 100644 index 000000000..0a8b746ed --- /dev/null +++ b/kubernetes/kubernetes/centos/files/add-option-to-disable-isolcpu-awareness.patch @@ -0,0 +1,79 @@ +From 8b765213a4e6d5cd4eecf361dadfec2851f1dd59 Mon Sep 17 00:00:00 2001 +From: Chris Friesen +Date: Fri, 23 Oct 2020 17:46:10 -0600 +Subject: [PATCH] enable support for kubernetes to ignore isolcpus + +The normal mechanisms for allocating isolated CPUs do not allow +a mix of isolated and exclusive CPUs in the same container. In +order to allow this in *very* limited cases where the pod spec +is known in advance we will add the ability to disable the normal +isolcpus behaviour. + +If the file "/etc/kubernetes/ignore_isolcpus" exists, then kubelet +will basically forget everything it knows about isolcpus and just +treat them like regular CPUs. + +The admin user can then rely on the fact that CPU allocation is +deterministic to ensure that the isolcpus they configure end up being +allocated to the correct pods. +--- + pkg/kubelet/cm/cpumanager/cpu_manager.go | 9 +++++++++ + pkg/kubelet/cm/cpumanager/policy_static.go | 8 ++++++++ + 2 files changed, 17 insertions(+) + +diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go +index c682f813..92992991 100644 +--- a/pkg/kubelet/cm/cpumanager/cpu_manager.go ++++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go +@@ -19,6 +19,7 @@ package cpumanager + import ( + "fmt" + "math" ++ "os" + "sync" + "time" + "strings" +@@ -56,6 +57,14 @@ const cpuManagerStateFileName = "cpu_manager_state" + + // get the system-level isolated CPUs + func getIsolcpus() cpuset.CPUSet { ++ ++ // This is a gross hack to basically turn off awareness of isolcpus to enable ++ // isolated cpus to be allocated to pods the same way as non-isolated CPUs. ++ if _, err := os.Stat("/etc/kubernetes/ignore_isolcpus"); err == nil { ++ klog.Infof("[cpumanager] turning off isolcpus awareness") ++ return cpuset.NewCPUSet() ++ } ++ + dat, err := ioutil.ReadFile("/sys/devices/system/cpu/isolated") + if err != nil { + klog.Errorf("[cpumanager] unable to read sysfs isolcpus subdir") +diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go +index 1913065e..4fb3202f 100644 +--- a/pkg/kubelet/cm/cpumanager/policy_static.go ++++ b/pkg/kubelet/cm/cpumanager/policy_static.go +@@ -18,6 +18,7 @@ package cpumanager + + import ( + "fmt" ++ "os" + "strconv" + + v1 "k8s.io/api/core/v1" +@@ -510,6 +511,13 @@ func isKubeInfra(pod *v1.Pod) bool { + + // get the isolated CPUs (if any) from the devices associated with a specific container + func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet { ++ ++ // This is a gross hack to basically turn off awareness of isolcpus to enable ++ // isolated cpus to be allocated to pods the same way as non-isolated CPUs. ++ if _, err := os.Stat("/etc/kubernetes/ignore_isolcpus"); err == nil { ++ return cpuset.NewCPUSet() ++ } ++ + // NOTE: This is required for TestStaticPolicyAdd() since makePod() does + // not create UID. We also need a way to properly stub devicemanager. + if len(string(pod.UID)) == 0 { +-- +2.16.6 + diff --git a/kubernetes/kubernetes/centos/kubernetes.spec b/kubernetes/kubernetes/centos/kubernetes.spec index 0cfb013e4..1e0f4321c 100644 --- a/kubernetes/kubernetes/centos/kubernetes.spec +++ b/kubernetes/kubernetes/centos/kubernetes.spec @@ -64,6 +64,7 @@ Patch6: kubelet-cpumanager-infrastructure-pods-use-system-re.patch Patch7: kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch Patch8: Fix-exclusive-CPU-allocations-being-deleted-at-conta.patch Patch9: kubeadm-create-platform-pods-with-zero-CPU-resources.patch +Patch10: add-option-to-disable-isolcpu-awareness.patch # It obsoletes cadvisor but needs its source code (literally integrated) Obsoletes: cadvisor @@ -852,6 +853,7 @@ Kubernetes client tools like kubectl %patch7 -p1 %patch8 -p1 %patch9 -p1 +%patch10 -p1 # copy contrib folder mkdir contrib