Create k8s-infra cgroup path before kubelet launch

This adds a kubelet ExecStartPre script to ensure cgroup is setup
prior to kubelet launch. This creates k8s-infra cgroup for a minimal
set of resource controllers, and configures cpuset attributes to span
all online cpus and nodes. This will do nothing if the k8s-infra cgroup
already exists (i.e., assume already configured).

NOTE: The creation of directories under /sys/fs/cgroup is volatile, and
does not persist reboots. The cpuset.mems and cpuset.cpus is later
updated by puppet kubernetes.pp manifest.

Tests performed:
Standard system: system install, lock/unlock controller & computes,
forced reboot: active/standby controller, computes.

Change-Id: I6a7aad5c40fe8225e9e16c8d8b40a0cffd76715d
Closes-Bug: 1828270
Signed-off-by: Jim Gauld <james.gauld@windriver.com>
This commit is contained in:
Jim Gauld 2019-05-10 12:57:37 -04:00
parent ce0cc60346
commit 6bd45c96dd
3 changed files with 120 additions and 0 deletions

View File

@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
EnvironmentFile=-/etc/sysconfig/kubelet EnvironmentFile=-/etc/sysconfig/kubelet
ExecStart= ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
ExecStopPost=/bin/rm -f /var/run/kubelet.pid ExecStopPost=/bin/rm -f /var/run/kubelet.pid
Restart=always Restart=always

View File

@ -0,0 +1,114 @@
#!/bin/bash
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This script does minimal cgroup setup for kubelet. This creates k8s-infra
# cgroup for a minimal set of resource controllers, and configures cpuset
# attributes to span all online cpus and nodes. This will do nothing if
# the k8s-infra cgroup already exists (i.e., assume already configured).
# NOTE: The creation of directories under /sys/fs/cgroup is volatile, and
# does not persist reboots. The cpuset.mems and cpuset.cpus is later updated
# by puppet kubernetes.pp manifest.
#
# Define minimal path
PATH=/bin:/usr/bin:/usr/local/bin
# Log info message to /var/log/daemon.log
function LOG {
logger -p daemon.info "$0($$): $@"
}
# Log error message to /var/log/daemon.log
function ERROR {
logger -s -p daemon.error "$0($$): ERROR: $@"
}
# Create minimal cgroup directories and configure cpuset attributes
function create_cgroup {
local cg_name=$1
local cg_nodeset=$2
local cg_cpuset=$3
local CGROUP=/sys/fs/cgroup
local CONTROLLERS=("cpuset" "memory" "cpu,cpuacct" "systemd")
local cnt=''
local CGDIR=''
local RC=0
# Create the cgroup for required controllers
for cnt in ${CONTROLLERS[@]}; do
CGDIR=${CGROUP}/${cnt}/${cg_name}
if [ -d ${CGDIR} ]; then
LOG "Nothing to do, already configured: ${CGDIR}."
exit ${RC}
fi
LOG "Creating: ${CGDIR}"
mkdir -p ${CGDIR}
RC=$?
if [ ${RC} -ne 0 ]; then
ERROR "Creating: ${CGDIR}, rc=${RC}"
exit ${RC}
fi
done
# Customize cpuset attributes
LOG "Configuring cgroup: ${cg_name}, nodeset: ${cg_nodeset}, cpuset: ${cg_cpuset}"
CGDIR=${CGROUP}/cpuset/${cg_name}
local CGMEMS=${CGDIR}/cpuset.mems
local CGCPUS=${CGDIR}/cpuset.cpus
local CGTASKS=${CGDIR}/tasks
# Assign cgroup memory nodeset
LOG "Assign nodeset ${cg_nodeset} to ${CGMEMS}"
/bin/echo ${cg_nodeset} > ${CGMEMS}
RC=$?
if [ ${RC} -ne 0 ]; then
ERROR "Unable to write to: ${CGMEMS}, rc=${RC}"
exit ${RC}
fi
# Assign cgroup cpus
LOG "Assign cpuset ${cg_cpuset} to ${CGCPUS}"
/bin/echo ${cg_cpuset} > ${CGCPUS}
RC=$?
if [ ${RC} -ne 0 ]; then
ERROR "Assigning: ${cg_cpuset} to ${CGCPUS}, rc=${RC}"
exit ${RC}
fi
# Set file ownership
chown root:root ${CGMEMS} ${CGCPUS} ${CGTASKS}
RC=$?
if [ ${RC} -ne 0 ]; then
ERROR "Setting owner for: ${CGMEMS}, ${CGCPUS}, ${CGTASKS}, rc=${RC}"
exit ${RC}
fi
# Set file mode permissions
chmod 644 ${CGMEMS} ${CGCPUS} ${CGTASKS}
RC=$?
if [ ${RC} -ne 0 ]; then
ERROR "Setting mode for: ${CGMEMS}, ${CGCPUS}, ${CGTASKS}, rc=${RC}"
exit ${RC}
fi
return ${RC}
}
if [ $UID -ne 0 ]; then
ERROR "Require sudo/root."
exit 1
fi
# Configure default kubepods cpuset to span all online cpus and nodes.
ONLINE_NODESET=$(/bin/cat /sys/devices/system/node/online)
ONLINE_CPUSET=$(/bin/cat /sys/devices/system/cpu/online)
# Configure kubelet cgroup to match cgroupRoot.
create_cgroup 'k8s-infra' ${ONLINE_NODESET} ${ONLINE_CPUSET}
exit $?

View File

@ -51,6 +51,7 @@ Source0: %{project}-v%{kube_version}.tar.gz
Source1: %{con_repo}-v%{con_commit}.tar.gz Source1: %{con_repo}-v%{con_commit}.tar.gz
Source3: kubernetes-accounting.conf Source3: kubernetes-accounting.conf
Source4: kubeadm.conf Source4: kubeadm.conf
Source5: kubelet-cgroup-setup.sh
Source33: genmanpages.sh Source33: genmanpages.sh
@ -914,6 +915,9 @@ install -p -m 755 -t %{buildroot}%{_bindir} ${output_path}/kubeadm
install -d -m 0755 %{buildroot}/%{_sysconfdir}/systemd/system/kubelet.service.d install -d -m 0755 %{buildroot}/%{_sysconfdir}/systemd/system/kubelet.service.d
install -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system/kubelet.service.d %{SOURCE4} install -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system/kubelet.service.d %{SOURCE4}
echo "+++ INSTALLING kubelet-cgroup-setup.sh"
install -p -m 0700 -t %{buildroot}/%{_bindir} %{SOURCE5}
binaries=(kube-controller-manager kube-scheduler kube-proxy kubelet kubectl) binaries=(kube-controller-manager kube-scheduler kube-proxy kubelet kubectl)
for bin in "${binaries[@]}"; do for bin in "${binaries[@]}"; do
echo "+++ HARDLINKING ${bin} to hyperkube" echo "+++ HARDLINKING ${bin} to hyperkube"
@ -1038,6 +1042,7 @@ fi
%{_mandir}/man1/kubelet.1* %{_mandir}/man1/kubelet.1*
%{_mandir}/man1/kube-proxy.1* %{_mandir}/man1/kube-proxy.1*
%{_bindir}/kubelet %{_bindir}/kubelet
%{_bindir}/kubelet-cgroup-setup.sh
%{_bindir}/kube-proxy %{_bindir}/kube-proxy
%{_bindir}/hyperkube %{_bindir}/hyperkube
%{_unitdir}/kube-proxy.service %{_unitdir}/kube-proxy.service