From 995e7e530d780fe06ae8ecffea2200168870a987 Mon Sep 17 00:00:00 2001 From: Jim Gauld Date: Fri, 11 Feb 2022 11:51:50 -0500 Subject: [PATCH] kubelet: sort isolcpus allocation when SMT enabled The existing device manager code returns CPUs as devices in unsorted order. This numerically sorts isolcpus allocations when SMT/HT is enabled on the host. This logs SMT pairs, singletons, and algorithm order details to make the algorithm understandable. Example log for a 3 cpu isolcpus request: 2022-02-11T16:27:50.345 controller-0 kubelet[1531574]: info I0211 16:27:50.345529 1531574 manager.go:741] order_devices_by_sibling: needed=3, smtpairs=[4 5 6 7 10 11], singletons=[8 12], order=[8 4 5 6 7 10 11 12] The specific host with SMT enabled has this topology: LOGICAL CPU TOPOLOGY: cpu_id : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 socket_id : 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 core_id : 0 0 1 1 2 2 3 3 4 4 5 5 6 6 7 7 thread_id : 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 Before cpu allocation, host has Isolated_free: 4-8,10-12. New pod gets the following isolcpus cpuset: 4-5,8. Test Plan: (On AIO-SX, SMT enabled) PASS: Verify cpu sort order for even needed and no singletons PASS: Verify cpu sort order for odd needed and no singletons PASS: Verify cpu sort order for even needed and singletons PASS: Verify cpu sort order for odd needed and singletons Story: 2008760 Task: 44190 Signed-off-by: Jim Gauld Change-Id: I1d743f80925b35ecee7936c12b0f4328f83b7eb2 --- ...isolcpus-allocation-when-SMT-enabled.patch | 50 +++++++++++++++++++ .../kubernetes-1.21.8/centos/kubernetes.spec | 2 + 2 files changed, 52 insertions(+) create mode 100644 kubernetes/kubernetes-1.21.8/centos/files/kubelet-sort-isolcpus-allocation-when-SMT-enabled.patch diff --git a/kubernetes/kubernetes-1.21.8/centos/files/kubelet-sort-isolcpus-allocation-when-SMT-enabled.patch b/kubernetes/kubernetes-1.21.8/centos/files/kubelet-sort-isolcpus-allocation-when-SMT-enabled.patch new file mode 100644 index 000000000..a58e47d5b --- /dev/null +++ b/kubernetes/kubernetes-1.21.8/centos/files/kubelet-sort-isolcpus-allocation-when-SMT-enabled.patch @@ -0,0 +1,50 @@ +From ba9ab333c8b7dca5252e604837914293dc232732 Mon Sep 17 00:00:00 2001 +From: Jim Gauld +Date: Fri, 11 Feb 2022 11:06:35 -0500 +Subject: [PATCH] kubelet: sort isolcpus allocation when SMT enabled + +The existing device manager code returns CPUs as devices in unsorted +order. This numerically sorts isolcpus allocations when SMT/HT is +enabled on the host. This logs SMT pairs, singletons, and algorithm +order details to make the algorithm understandable. + +Signed-off-by: Jim Gauld +--- + pkg/kubelet/cm/devicemanager/manager.go | 13 ++++++++++++- + 1 file changed, 12 insertions(+), 1 deletion(-) + +diff --git a/pkg/kubelet/cm/devicemanager/manager.go b/pkg/kubelet/cm/devicemanager/manager.go +index 609da8ed86b..a4b247714f7 100644 +--- a/pkg/kubelet/cm/devicemanager/manager.go ++++ b/pkg/kubelet/cm/devicemanager/manager.go +@@ -686,7 +686,16 @@ func order_devices_by_sibling(devices sets.String, needed int) ([]string, error) + return cpu_lst[0] + } + } ++ //Make post-analysis of selection algorithm obvious by numerical sorting ++ //the available isolated cpu_id. ++ cpu_ids := make([]int, 0, int(devices.Len())) + for cpu_id := range devices { ++ cpu_id_, _ := strconv.Atoi(cpu_id) ++ cpu_ids = append(cpu_ids, cpu_id_) ++ } ++ sort.Ints(cpu_ids) ++ for _, _cpu_id := range cpu_ids { ++ cpu_id := strconv.Itoa(_cpu_id) + // If we've already found cpu_id as a sibling, skip it. + if _, ok := _iterated_cpu[cpu_id]; ok { + continue +@@ -728,7 +737,9 @@ func order_devices_by_sibling(devices sets.String, needed int) ([]string, error) + } + } + } +- //klog.Infof("needed=%d ordered_cpu_list=%v", needed, dev_lst) ++ //This algorithm will get some attention. Show minimal details. ++ klog.Infof("order_devices_by_sibling: needed=%d, smtpairs=%v, singletons=%v, order=%v", ++ needed, sibling_lst, single_lst, dev_lst) + return dev_lst, nil + } + func smt_enabled() bool { +-- +2.25.1 + diff --git a/kubernetes/kubernetes-1.21.8/centos/kubernetes.spec b/kubernetes/kubernetes-1.21.8/centos/kubernetes.spec index a6c9f80f3..d1077998c 100644 --- a/kubernetes/kubernetes-1.21.8/centos/kubernetes.spec +++ b/kubernetes/kubernetes-1.21.8/centos/kubernetes.spec @@ -62,6 +62,7 @@ Patch6: enable-support-for-kubernetes-to-ignore-isolcpus.patch Patch7: Revert-use-subpath-for-coredns-only-for-default-repo.patch Patch8: Change-log-level-to-Debug.patch Patch9: kubernetes-make-isolcpus-allocation-SMT-aware.patch +Patch10: kubelet-sort-isolcpus-allocation-when-SMT-enabled.patch # It obsoletes cadvisor but needs its source code (literally integrated) Obsoletes: cadvisor @@ -858,6 +859,7 @@ install in production. %patch7 -p1 %patch8 -p1 %patch9 -p1 +%patch10 -p1 #src/k8s.io/kubernetes/pkg/util/certificates # Patch the code to remove eliptic.P224 support