Merge "AIO reaffine tasks and k8s-infra during startup"
This commit is contained in:
commit
4fe2b6bed5
|
@ -16,10 +16,12 @@ class platform::compute::config
|
||||||
content => template('platform/worker_reserved.conf.erb')
|
content => template('platform/worker_reserved.conf.erb')
|
||||||
}
|
}
|
||||||
|
|
||||||
file { '/etc/systemd/system.conf.d/platform-cpuaffinity.conf':
|
if $::platform::params::system_type != 'All-in-one' {
|
||||||
ensure => 'present',
|
file { '/etc/systemd/system.conf.d/platform-cpuaffinity.conf':
|
||||||
replace => true,
|
ensure => 'present',
|
||||||
content => template('platform/systemd-system-cpuaffinity.conf.erb')
|
replace => true,
|
||||||
|
content => template('platform/systemd-system-cpuaffinity.conf.erb')
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -13,3 +13,4 @@ restarts = 3 ; restarts before error assertion
|
||||||
startuptime = 5 ; seconds to wait after process start
|
startuptime = 5 ; seconds to wait after process start
|
||||||
interval = 5 ; number of seconds to wait between restarts
|
interval = 5 ; number of seconds to wait between restarts
|
||||||
debounce = 20 ; number of seconds to wait before degrade clear
|
debounce = 20 ; number of seconds to wait before degrade clear
|
||||||
|
subfunction = last-config ; run monitor only after last config is run
|
||||||
|
|
|
@ -189,8 +189,7 @@ class KubernetesPuppet(base.BasePuppet):
|
||||||
# TODO(jgauld): Commented out for now, using host_cpuset instead.
|
# TODO(jgauld): Commented out for now, using host_cpuset instead.
|
||||||
# nonplatform_cpuset = host_cpuset - platform_cpuset
|
# nonplatform_cpuset = host_cpuset - platform_cpuset
|
||||||
|
|
||||||
if constants.WORKER in utils.get_personalities(host) \
|
if constants.WORKER in utils.get_personalities(host):
|
||||||
and constants.CONTROLLER not in utils.get_personalities(host):
|
|
||||||
if self.is_openstack_compute(host):
|
if self.is_openstack_compute(host):
|
||||||
k8s_cpuset = utils.format_range_set(platform_cpuset)
|
k8s_cpuset = utils.format_range_set(platform_cpuset)
|
||||||
k8s_nodeset = utils.format_range_set(platform_nodeset)
|
k8s_nodeset = utils.format_range_set(platform_nodeset)
|
||||||
|
|
|
@ -41,19 +41,6 @@ function affine_tasks {
|
||||||
local PIDLIST
|
local PIDLIST
|
||||||
local RET=0
|
local RET=0
|
||||||
|
|
||||||
# Affine non-kernel-thread tasks (excluded [kthreadd] and its children) to all available
|
|
||||||
# cores. They will be reaffined to platform cores later on as part of nova-compute
|
|
||||||
# launch.
|
|
||||||
##log_debug "Affining all tasks to all available CPUs..."
|
|
||||||
# TODO: Should revisit this since this leaves a few lingering floating
|
|
||||||
# tasks and does not really work with cgroup cpusets.
|
|
||||||
# Comment out for now. Cleanup required.
|
|
||||||
##affine_tasks_to_all_cores
|
|
||||||
##RET=$?
|
|
||||||
##if [ $RET -ne 0 ]; then
|
|
||||||
## log_error "Some tasks failed to be affined to all cores."
|
|
||||||
##fi
|
|
||||||
|
|
||||||
# Get number of logical cpus
|
# Get number of logical cpus
|
||||||
N_CPUS=$(cat /proc/cpuinfo 2>/dev/null | \
|
N_CPUS=$(cat /proc/cpuinfo 2>/dev/null | \
|
||||||
awk '/^[pP]rocessor/ { n +=1 } END { print (n>0) ? n : 1}')
|
awk '/^[pP]rocessor/ { n +=1 } END { print (n>0) ? n : 1}')
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=StarlingX Affine Tasks
|
Description=StarlingX Affine Tasks
|
||||||
After=syslog.service network.service dbus.service sw-patch.service affine-platform.sh.service
|
After=syslog.service network.service dbus.service sw-patch.service affine-platform.sh.service
|
||||||
Before=kubelet.service
|
Before=workerconfig.service
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=oneshot
|
Type=simple
|
||||||
ExecStart=/etc/init.d/affine-tasks.sh start
|
ExecStart=/etc/init.d/affine-tasks.sh start
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
WantedBy=multi-user.target
|
WantedBy=multi-user.target
|
||||||
|
|
|
@ -1,62 +1,441 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
###############################################################################
|
#
|
||||||
# Copyright (c) 2019 Wind River Systems, Inc.
|
# Copyright (c) 2019 Wind River Systems, Inc.
|
||||||
#
|
#
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
#
|
#
|
||||||
###############################################################################
|
|
||||||
#
|
#
|
||||||
|
# chkconfig: 2345 80 80
|
||||||
|
#
|
||||||
|
|
||||||
### BEGIN INIT INFO
|
### BEGIN INIT INFO
|
||||||
# Provides: affine-tasks
|
# Provides: affine-tasks
|
||||||
# Required-Start:
|
# Required-Start:
|
||||||
# Required-Stop:
|
# Required-Stop:
|
||||||
# Default-Start: 2 3 4 5
|
# Default-Start: 2 3 4 5
|
||||||
# Default-Stop: 0 1 6
|
# Default-Stop: 0 1 6
|
||||||
# Short-Description: affine tasks
|
# Short-Description: reaffine tasks on AIO
|
||||||
# Description: This script will affine tasks to the platform cores of the
|
# Description: This script will dynamically reaffine tasks
|
||||||
# host. This ensures that system processes are constrained to platform
|
# and k8s-infra cgroup cpuset on AIO nodes only. This accomodates
|
||||||
# cores and will not run on cores with VMs/containers.
|
# CPU intensive phases of work. Tasks are initially allowed to float
|
||||||
|
# across all cores. Once system is at steady-state, this will ensure
|
||||||
|
# that K8S pods are constrained to platform cores and do not run on
|
||||||
|
# cores with VMs/containers.
|
||||||
### END INIT INFO
|
### END INIT INFO
|
||||||
|
#
|
||||||
|
# Background:
|
||||||
|
# There is significant parallel CPU intensive activity:
|
||||||
|
# - during stx-application apply before critical openstack pods are running,
|
||||||
|
# e.g., to download docker images, and start all pods.
|
||||||
|
# - during init and pod recovery after reboot or DOR.
|
||||||
|
#
|
||||||
|
# This enables use of all cpus during CPU intensive phase, otherwise the
|
||||||
|
# startup processing time is considerably longer and we easily hit timeout.
|
||||||
|
#
|
||||||
|
# This script waits forever for sufficient platform readiness criteria
|
||||||
|
# (e.g., system critical pods are recovered, nova-compute is running,
|
||||||
|
# cinder-volume is running, openstack pods are running), and we have waited
|
||||||
|
# a short stabilization period before reaffining to the platform cpus.
|
||||||
|
#
|
||||||
|
# NOTE: child cgroup cpuset and nodeset must be a subset of the parent
|
||||||
|
# cgroup's attributes. This requires traversing the tree hierachy in
|
||||||
|
# specific order when dynamically modifying these attributes.
|
||||||
|
#
|
||||||
|
################################################################################
|
||||||
|
# Define minimal path
|
||||||
|
PATH=/bin:/usr/bin:/usr/local/bin
|
||||||
|
|
||||||
|
CPUMAP_FUNCTIONS=${CPUMAP_FUNCTIONS:-"/etc/init.d/cpumap_functions.sh"}
|
||||||
|
[[ -e ${CPUMAP_FUNCTIONS} ]] && source ${CPUMAP_FUNCTIONS}
|
||||||
|
|
||||||
. /usr/bin/tsconfig
|
# Bring in platform definitions
|
||||||
. /etc/init.d/task_affinity_functions.sh
|
. /etc/platform/platform.conf
|
||||||
|
|
||||||
log ()
|
# Environment for kubectl
|
||||||
{
|
export KUBECONFIG=/etc/kubernetes/admin.conf
|
||||||
logger -p local1.info -t affine_tasks $@
|
|
||||||
echo affine_tasks: "$@"
|
# Global parameters
|
||||||
|
CGDIR_K8S=/sys/fs/cgroup/cpuset/k8s-infra
|
||||||
|
INIT_INTERVAL_SECONDS=10
|
||||||
|
CHECK_INTERVAL_SECONDS=30
|
||||||
|
PRINT_INTERVAL_SECONDS=300
|
||||||
|
STABILIZATION_SECONDS=150
|
||||||
|
|
||||||
|
# Define pidfile
|
||||||
|
LNAME=$(readlink -n -f $0)
|
||||||
|
NAME=$(basename $LNAME)
|
||||||
|
PIDFILE=/var/run/${NAME}.pid
|
||||||
|
|
||||||
|
# Define number of logical cpus
|
||||||
|
LOGICAL_CPUS=$(getconf _NPROCESSORS_ONLN)
|
||||||
|
|
||||||
|
# Define the memory nodeset and cpuset that span all online cpus and nodes
|
||||||
|
ONLINE_NODES=$(/bin/cat /sys/devices/system/node/online)
|
||||||
|
ONLINE_CPUS=$(/bin/cat /sys/devices/system/cpu/online)
|
||||||
|
ONLINE_MASK=$(cpulist_to_cpumap ${ONLINE_CPUS} ${LOGICAL_CPUS} | \
|
||||||
|
awk '{print tolower($0)}')
|
||||||
|
|
||||||
|
ISOL_CPUS=$(/bin/cat /sys/devices/system/cpu/isolated)
|
||||||
|
if [ ! -z "${ISOL_CPUS}" ]; then
|
||||||
|
ISOL_CPUMAP=$(cpulist_to_cpumap ${ISOL_CPUS} ${LOGICAL_CPUS})
|
||||||
|
NONISOL_CPUMAP=$(invert_cpumap ${ISOL_CPUMAP} ${LOGICAL_CPUS})
|
||||||
|
NONISOL_CPUS=$(cpumap_to_cpulist ${NONISOL_CPUMAP} ${LOGICAL_CPUS})
|
||||||
|
NONISOL_MASK=$(cpulist_to_cpumap ${NONISOL_CPUS} ${LOGICAL_CPUS} | \
|
||||||
|
awk '{print tolower($0)}')
|
||||||
|
else
|
||||||
|
ISOL_CPUMAP='0'
|
||||||
|
NONISOL_CPUS=${ONLINE_CPUS}
|
||||||
|
NONISOL_MASK=${ONLINE_MASK}
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Define platform memory nodeset and cpuset
|
||||||
|
PLATFORM_NODES=$(cat /sys/devices/system/node/online)
|
||||||
|
PLATFORM_CPUS=$(platform_expanded_cpu_list)
|
||||||
|
|
||||||
|
# Global variables
|
||||||
|
NOT_READY_REASON=""
|
||||||
|
STABLE=0
|
||||||
|
|
||||||
|
# Log info message to /var/log/daemon.log
|
||||||
|
function LOG {
|
||||||
|
logger -p daemon.info -t "${NAME}($$): " "$@"
|
||||||
}
|
}
|
||||||
|
|
||||||
start ()
|
# Log error message to /var/log/daemon.log
|
||||||
{
|
function ERROR {
|
||||||
log "Starting affine_tasks. Reaffining tasks to platform cores..."
|
logger -s -p daemon.error -t "${NAME}($$): " "$@"
|
||||||
if [ ! -f ${INITIAL_CONFIG_COMPLETE_FLAG} ]; then
|
}
|
||||||
log "Initial Configuration incomplete. Skipping affining tasks."
|
|
||||||
exit 0
|
# Update cgroup k8s-infra cpuset and nodeset to span all non-isolated cpus.
|
||||||
|
function update_cgroup_cpuset_k8s_infra_all {
|
||||||
|
# Set all cgroup cpuset and nodeset in tree hierarchy order.
|
||||||
|
# This will always work, no matter the previous cpuset state.
|
||||||
|
find ${CGDIR_K8S} -type d | \
|
||||||
|
while read d; do
|
||||||
|
/bin/echo ${ONLINE_NODES} > ${d}/cpuset.mems 2>/dev/null
|
||||||
|
/bin/echo ${NONISOL_CPUS} > ${d}/cpuset.cpus 2>/dev/null
|
||||||
|
done
|
||||||
|
LOG "Update ${CGDIR_K8S}," \
|
||||||
|
"ONLINE_NODES=${ONLINE_NODES}, NONISOL_CPUS=${NONISOL_CPUS}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Update cgroup k8s-infra to span platform cpuset and nodeset.
|
||||||
|
function update_cgroup_cpuset_k8s_infra_platform {
|
||||||
|
# Clear any existing cpuset settings. This ensures that the
|
||||||
|
# subsequent shrink to platform cpuset will always work.
|
||||||
|
update_cgroup_cpuset_k8s_infra_all
|
||||||
|
|
||||||
|
# Set all cgroup cpuset and nodeset in depth-first order.
|
||||||
|
# NOTE: this only works if we are shrinking the cpuset.
|
||||||
|
find ${CGDIR_K8S} -depth -type d | \
|
||||||
|
while read d; do
|
||||||
|
/bin/echo ${PLATFORM_NODES} > ${d}/cpuset.mems 2>/dev/null
|
||||||
|
/bin/echo ${PLATFORM_CPUS} > ${d}/cpuset.cpus 2>/dev/null
|
||||||
|
done
|
||||||
|
LOG "Update ${CGDIR_K8S}," \
|
||||||
|
"PLATFORM_NODES=${PLATFORM_NODES}, PLATFORM_CPUS=${PLATFORM_CPUS}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Check criteria for K8s platform ready on this node.
|
||||||
|
# i.e., k8s-infra is configured, kubelet is running
|
||||||
|
function is_k8s_platform_ready {
|
||||||
|
local PASS=0
|
||||||
|
local FAIL=1
|
||||||
|
|
||||||
|
# Global variable
|
||||||
|
NOT_READY_REASON=""
|
||||||
|
|
||||||
|
# Check that cgroup cpuset k8s-infra has been configured
|
||||||
|
if [ ! -e ${CGDIR_K8S} ]; then
|
||||||
|
NOT_READY_REASON="k8s-infra not configured"
|
||||||
|
return ${FAIL}
|
||||||
fi
|
fi
|
||||||
# TODO: Should revisit this since this leaves a few lingering floating
|
|
||||||
# tasks and does not really work with cgroup cpusets.
|
# Check that kubelet is running and stable
|
||||||
# Comment out for now. Cleanup required.
|
if systemctl is-active kubelet --quiet; then
|
||||||
##affine_tasks_to_platform_cores
|
PID=$(systemctl show kubelet.service -p MainPID | \
|
||||||
##[[ $? -eq 0 ]] && log "Tasks re-affining done." || log "Tasks re-affining failed."
|
awk -vFS='=' '{print $2}')
|
||||||
|
if [ ${PID} -eq 0 ]; then
|
||||||
|
NOT_READY_REASON="kubelet not running"
|
||||||
|
return ${FAIL}
|
||||||
|
fi
|
||||||
|
up=$(ps -p ${PID} -o etimes= 2>/dev/null | awk '{print $1}')
|
||||||
|
if ! { [ -n "${up}" -a ${up} -ge 30 ]; }
|
||||||
|
then
|
||||||
|
NOT_READY_REASON="kubelet not yet stable"
|
||||||
|
return ${FAIL}
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
NOT_READY_REASON="kubelet not running"
|
||||||
|
return ${FAIL}
|
||||||
|
fi
|
||||||
|
|
||||||
|
LOG "kubelet is ready"
|
||||||
|
return ${PASS}
|
||||||
}
|
}
|
||||||
|
|
||||||
stop ()
|
# Determine whether this node has 'static' cpu manager policy.
|
||||||
{
|
# NOTE: This check assumes that kubelet is already running locally.
|
||||||
log "Stopping affine_tasks..."
|
function is_static_cpu_manager_policy {
|
||||||
|
local PASS=0
|
||||||
|
local FAIL=1
|
||||||
|
|
||||||
|
state=$(cat /var/lib/kubelet/cpu_manager_state 2>/dev/null)
|
||||||
|
if [[ $state =~ \"policyName\":.?\"static\" ]]; then
|
||||||
|
return ${PASS}
|
||||||
|
else
|
||||||
|
return ${FAIL}
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
status()
|
# Check criteria for K8s platform steady-state ready on this node.
|
||||||
{
|
# i.e., kube-system pods have recovered, kube application apply
|
||||||
|
# has completed, nova-compute is running, cinder-volume is running.
|
||||||
|
# NOTE: This function depends on kubectl commands, so is only
|
||||||
|
# usable on controllers.
|
||||||
|
function is_k8s_platform_steady_state_ready {
|
||||||
|
local PASS=0
|
||||||
|
local FAIL=1
|
||||||
|
local this_node=${HOSTNAME}
|
||||||
|
|
||||||
|
# Global variable
|
||||||
|
NOT_READY_REASON=""
|
||||||
|
|
||||||
|
# Check that kube-system pods have recovered on this node
|
||||||
|
npods=$(kubectl get pods --namespace kube-system --no-headers \
|
||||||
|
--field-selector spec.nodeName=${this_node} 2>/dev/null | \
|
||||||
|
awk '
|
||||||
|
BEGIN { n=0; }
|
||||||
|
!/Completed|Running/ { n+=1 }
|
||||||
|
END { printf "%d\n", n; }
|
||||||
|
')
|
||||||
|
if [ ${npods} -gt 0 ]; then
|
||||||
|
NOT_READY_REASON="${npods} kube-system pods not recovered"
|
||||||
|
STABLE=0
|
||||||
|
return ${FAIL}
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Wait for a few critical openstack pods to be running if this is
|
||||||
|
# an openstack-compute-node. This is not an exhaustive list.
|
||||||
|
# Make sure that all openstack pods on this node are running.
|
||||||
|
labels=$(kubectl get node ${this_node} \
|
||||||
|
--no-headers --show-labels 2>/dev/null | awk '{print $NF}')
|
||||||
|
if [[ $labels =~ openstack-compute-node=enabled ]]; then
|
||||||
|
# nova-compute is one of the last charts to recover after reboot
|
||||||
|
PODS=( $(kubectl get pods --namespace openstack --no-headers \
|
||||||
|
--selector application=nova,component=compute \
|
||||||
|
--field-selector \
|
||||||
|
spec.nodeName=${this_node},status.phase=Running 2>/dev/null) )
|
||||||
|
if [ ${#PODS[@]} -eq 0 ]; then
|
||||||
|
NOT_READY_REASON="nova-compute pod not running"
|
||||||
|
STABLE=0
|
||||||
|
return ${FAIL}
|
||||||
|
fi
|
||||||
|
|
||||||
|
# cinder-volume is one of the last charts to recover after reboot
|
||||||
|
PODS=( $(kubectl get pods --namespace openstack --no-headers \
|
||||||
|
--selector application=cinder,component=volume \
|
||||||
|
--field-selector \
|
||||||
|
spec.nodeName=${this_node},status.phase=Running 2>/dev/null) )
|
||||||
|
if [ ${#PODS[@]} -eq 0 ]; then
|
||||||
|
NOT_READY_REASON="cinder-volume pod not running"
|
||||||
|
STABLE=0
|
||||||
|
return ${FAIL}
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check that all openstack pods on this node have recovered
|
||||||
|
npods=$(kubectl get pods --namespace openstack --no-headers \
|
||||||
|
--field-selector spec.nodeName=${this_node} 2>/dev/null | \
|
||||||
|
awk '
|
||||||
|
BEGIN { n=0; }
|
||||||
|
!/Completed|Running/ { n+=1 }
|
||||||
|
END { printf "%d\n", n; }
|
||||||
|
')
|
||||||
|
if [ ${npods} -gt 0 ]; then
|
||||||
|
NOT_READY_REASON="${npods} openstack pods not recovered"
|
||||||
|
STABLE=0
|
||||||
|
return ${FAIL}
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Evaluate elapsed time since check criteria pass
|
||||||
|
if [ ${STABLE} -eq 0 ]; then
|
||||||
|
STABLE=${SECONDS}
|
||||||
|
fi
|
||||||
|
dt=$(( ${SECONDS} - ${STABLE} ))
|
||||||
|
if [ ${dt} -lt ${STABILIZATION_SECONDS} ]; then
|
||||||
|
NOT_READY_REASON="stabilization wait"
|
||||||
|
return ${FAIL}
|
||||||
|
fi
|
||||||
|
|
||||||
|
LOG "K8S is ready"
|
||||||
|
return ${PASS}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Return list of reaffineable pids. This includes all processes, but excludes
|
||||||
|
# kernel threads, vSwitch, and anything in K8S or qemu/kvm.
|
||||||
|
function reaffineable_pids {
|
||||||
|
local pids_excl
|
||||||
|
local pidlist
|
||||||
|
|
||||||
|
pids_excl=$(ps -eL -o pid=,comm= | \
|
||||||
|
awk -vORS=',' '/eal-intr-thread|kthreadd/ {print $1}' | \
|
||||||
|
sed 's/,$/\n/')
|
||||||
|
pidlist=$(ps --ppid ${pids_excl} -p ${pids_excl} --deselect \
|
||||||
|
-o pid=,cgroup= | \
|
||||||
|
awk '!/k8s-infra|machine.slice/ {print $1; }')
|
||||||
|
echo "${pidlist[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
function affine_tasks_to_all_cores {
|
||||||
|
local pidlist
|
||||||
|
local count=0
|
||||||
|
|
||||||
|
LOG "Affine all tasks, CPUS: ${NONISOL_CPUS};" \
|
||||||
|
"online=${ONLINE_CPUS} (0x${ONLINE_MASK})," \
|
||||||
|
"isol=${ISOL_CPUS}, nonisol=${NONISOL_CPUS} (0x${NONISOL_MASK})"
|
||||||
|
|
||||||
|
pidlist=( $(reaffineable_pids) )
|
||||||
|
for pid in ${pidlist[@]}; do
|
||||||
|
count=$((${count} + 1))
|
||||||
|
taskset --all-tasks --pid --cpu-list \
|
||||||
|
${NONISOL_CPUS} ${pid} > /dev/null 2>&1
|
||||||
|
done
|
||||||
|
|
||||||
|
LOG "Affined ${count} processes to all cores."
|
||||||
|
}
|
||||||
|
|
||||||
|
function affine_tasks_to_platform_cores {
|
||||||
|
local pidlist
|
||||||
|
local count=0
|
||||||
|
|
||||||
|
LOG "Affine all tasks, PLATFORM_CPUS=${PLATFORM_CPUS}"
|
||||||
|
|
||||||
|
pidlist=( $(reaffineable_pids) )
|
||||||
|
for pid in ${pidlist[@]}; do
|
||||||
|
pid_mask=$(taskset -p $pid 2> /dev/null | awk '{print $6}')
|
||||||
|
if [ "${pid_mask}" == "${NONISOL_MASK}" ]; then
|
||||||
|
count=$((${count} + 1))
|
||||||
|
taskset --all-tasks --pid --cpu-list \
|
||||||
|
${PLATFORM_CPUS} ${pid} > /dev/null 2>&1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Reaffine vSwitch tasks that span multiple cpus to platform cpus
|
||||||
|
pidlist=$(ps -eL -o pid=,comm= | awk '/eal-intr-thread/ {print $1}')
|
||||||
|
for pid in ${pidlist[@]}; do
|
||||||
|
count=$((${count} + 1))
|
||||||
|
grep Cpus_allowed_list /proc/${pid}/task/*/status 2>/dev/null | \
|
||||||
|
sed 's#/# #g' | awk '/,|-/ {print $4}' | \
|
||||||
|
xargs --no-run-if-empty -i{} \
|
||||||
|
taskset --pid --cpu-list ${PLATFORM_CPUS} {} > /dev/null 2>&1
|
||||||
|
done
|
||||||
|
|
||||||
|
LOG "Affined ${count} processes to platform cores."
|
||||||
|
}
|
||||||
|
|
||||||
|
function start {
|
||||||
|
# Ensure this only runs on AIO
|
||||||
|
if ! { [[ "$nodetype" = "controller" ]] && [[ $subfunction = *worker* ]]; }
|
||||||
|
then
|
||||||
|
LOG "Not AIO, nothing to do."
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Abort if another instantiation is already running
|
||||||
|
if [ -e ${PIDFILE} ]; then
|
||||||
|
PID=$(cat ${PIDFILE})
|
||||||
|
if [ -n "${PID}" -a -e /proc/${PID} ]; then
|
||||||
|
ERROR "Aborting, ${PID} already running: ${PIDFILE}."
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
OUT=$(rm -v -f ${PIDFILE})
|
||||||
|
LOG "${OUT}"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
LOG "Starting."
|
||||||
|
|
||||||
|
# Create pidfile to indicate the script is running
|
||||||
|
echo $$ > ${PIDFILE}
|
||||||
|
|
||||||
|
# Affine all tasks to float on all cores
|
||||||
|
affine_tasks_to_all_cores
|
||||||
|
|
||||||
|
# Wait for kubelet to be running
|
||||||
|
t0=${SECONDS}
|
||||||
|
until is_k8s_platform_ready; do
|
||||||
|
dt=$(( ${SECONDS} - ${t0} ))
|
||||||
|
if [ ${dt} -ge ${PRINT_INTERVAL_SECONDS} ]; then
|
||||||
|
t0=${SECONDS}
|
||||||
|
LOG "Recovery wait, elapsed ${SECONDS} seconds." \
|
||||||
|
"Reason: ${NOT_READY_REASON}"
|
||||||
|
fi
|
||||||
|
sleep ${INIT_INTERVAL_SECONDS}
|
||||||
|
done
|
||||||
|
|
||||||
|
# Update K8S cpuset so that pods float on all cpus
|
||||||
|
# NOTE: dynamic cpuset changes incompatible with static policy
|
||||||
|
if ! is_static_cpu_manager_policy; then
|
||||||
|
update_cgroup_cpuset_k8s_infra_all
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Wait until K8s pods have recovered and nova-compute is running
|
||||||
|
t0=${SECONDS}
|
||||||
|
until is_k8s_platform_steady_state_ready; do
|
||||||
|
dt=$(( ${SECONDS} - ${t0} ))
|
||||||
|
if [ ${dt} -ge ${PRINT_INTERVAL_SECONDS} ]; then
|
||||||
|
t0=${SECONDS}
|
||||||
|
LOG "Recovery wait, elapsed ${SECONDS} seconds." \
|
||||||
|
"Reason: ${NOT_READY_REASON}"
|
||||||
|
fi
|
||||||
|
sleep ${CHECK_INTERVAL_SECONDS}
|
||||||
|
done
|
||||||
|
|
||||||
|
# Update K8S cpuset to platform cores
|
||||||
|
if ! is_static_cpu_manager_policy; then
|
||||||
|
update_cgroup_cpuset_k8s_infra_platform
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Affine all floating tasks back to platform cores
|
||||||
|
affine_tasks_to_platform_cores
|
||||||
|
|
||||||
|
# Remove pidfile after successful completion
|
||||||
|
rm -f ${PIDFILE}
|
||||||
|
|
||||||
|
LOG "Complete."
|
||||||
|
}
|
||||||
|
|
||||||
|
function stop {
|
||||||
|
LOG "Stopping."
|
||||||
|
|
||||||
|
# Forcibly stop any running instantiation
|
||||||
|
if [ -e ${PIDFILE} ]; then
|
||||||
|
PID=$(cat ${PIDFILE})
|
||||||
|
if [ -n "${PID}" -a -e /proc/${PID} ]; then
|
||||||
|
LOG "Stopping ${PID}: ${PIDFILE}."
|
||||||
|
kill -9 ${PID}
|
||||||
|
timeout 20 tail --pid=${PID} -f /dev/null
|
||||||
|
fi
|
||||||
|
OUT=$(rm -v -f ${PIDFILE})
|
||||||
|
LOG "${OUT}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function status {
|
||||||
:
|
:
|
||||||
}
|
}
|
||||||
|
|
||||||
reset()
|
function reset {
|
||||||
{
|
|
||||||
:
|
:
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if [ ${UID} -ne 0 ]; then
|
||||||
|
ERROR "Need sudo/root permission."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
case "$1" in
|
case "$1" in
|
||||||
start)
|
start)
|
||||||
start
|
start
|
||||||
|
|
|
@ -29,19 +29,22 @@ LOG_DEBUG=1
|
||||||
TAG="TASKAFFINITY:"
|
TAG="TASKAFFINITY:"
|
||||||
|
|
||||||
TASK_AFFINING_INCOMPLETE="/etc/platform/.task_affining_incomplete"
|
TASK_AFFINING_INCOMPLETE="/etc/platform/.task_affining_incomplete"
|
||||||
N_CPUS=$(cat /proc/cpuinfo 2>/dev/null | \
|
N_CPUS=$(getconf _NPROCESSORS_ONLN)
|
||||||
awk '/^[pP]rocessor/ { n +=1 } END { print (n>0) ? n : 1}')
|
|
||||||
FULLSET_CPUS="0-"$((N_CPUS-1))
|
FULLSET_CPUS="0-"$((N_CPUS-1))
|
||||||
FULLSET_MASK=$(cpulist_to_cpumap ${FULLSET_CPUS} ${N_CPUS})
|
FULLSET_MASK=$(cpulist_to_cpumap ${FULLSET_CPUS} ${N_CPUS})
|
||||||
PLATFORM_CPUS=$(get_platform_cpu_list)
|
PLATFORM_CPUS=$(platform_expanded_cpu_list)
|
||||||
PLATFORM_CPULIST=$(get_platform_cpu_list| \
|
PLATFORM_CPULIST=$(platform_expanded_cpu_list| \
|
||||||
perl -pe 's/(\d+)-(\d+)/join(",",$1..$2)/eg'| \
|
perl -pe 's/(\d+)-(\d+)/join(",",$1..$2)/eg'| \
|
||||||
sed 's/,/ /g')
|
sed 's/,/ /g')
|
||||||
VSWITCH_CPULIST=$(get_vswitch_cpu_list| \
|
VSWITCH_CPULIST=$(get_vswitch_cpu_list| \
|
||||||
perl -pe 's/(\d+)-(\d+)/join(",",$1..$2)/eg'| \
|
perl -pe 's/(\d+)-(\d+)/join(",",$1..$2)/eg'| \
|
||||||
sed 's/,/ /g')
|
sed 's/,/ /g')
|
||||||
|
if [[ $vswitch_type =~ none ]]; then
|
||||||
|
VSWITCH_CPULIST=""
|
||||||
|
fi
|
||||||
|
|
||||||
IDLE_MARK=95.0
|
IDLE_MARK=95.0
|
||||||
KERNEL=`uname -a`
|
KERNEL=$(uname -a)
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
# Check if a given core is one of the platform cores
|
# Check if a given core is one of the platform cores
|
||||||
|
@ -69,98 +72,19 @@ function is_vswitch_core {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
################################################################################
|
# Return list of reaffineable pids. This includes all processes, but excludes
|
||||||
# An audit and corrective action following a swact
|
# kernel threads, vSwitch, and anything in K8S or qemu/kvm.
|
||||||
################################################################################
|
function reaffineable_pids {
|
||||||
function audit_and_reaffine {
|
local pids_excl
|
||||||
local mask=$1
|
|
||||||
local cmd_str=""
|
|
||||||
local tasklist
|
|
||||||
|
|
||||||
cmd_str="ps-sched.sh|awk '(\$9==\"$mask\") {print \$2}'"
|
|
||||||
|
|
||||||
tasklist=($(eval $cmd_str))
|
|
||||||
# log_debug "cmd str = $cmd_str"
|
|
||||||
log_debug "${TAG} There are ${#tasklist[@]} tasks to reaffine."
|
|
||||||
|
|
||||||
for task in ${tasklist[@]}; do
|
|
||||||
taskset -acp ${PLATFORM_CPUS} $task &> /dev/null
|
|
||||||
rc=$?
|
|
||||||
[[ $rc -ne 0 ]] && log_error "Failed to set CPU affinity for pid $pid, rc=$rc"
|
|
||||||
done
|
|
||||||
tasklist=($(eval $cmd_str))
|
|
||||||
[[ ${#tasklist[@]} -eq 0 ]] && return 0 || return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
# The following function is used to verify that any sleeping management tasks
|
|
||||||
# that are on non-platform cores can be migrated to platform cores as soon as
|
|
||||||
# they are scheduled. It can be invoked either manually or from goenableCompute
|
|
||||||
# script as a scheduled job (with a few minute delay) if desired.
|
|
||||||
# The induced tasks migration should be done after all VMs have been restored
|
|
||||||
# following a host reboot in AIO, hence the delay.
|
|
||||||
################################################################################
|
|
||||||
function move_inactive_threads_to_platform_cores {
|
|
||||||
local tasklist
|
|
||||||
local cmd_str=""
|
|
||||||
|
|
||||||
# Compile a list of non-kernel & non-vswitch/VM related threads that are not
|
|
||||||
# on platform cores.
|
|
||||||
# e.g. if the platform cpulist value is "0 8", the resulting command to be
|
|
||||||
# evaluated should look like this:
|
|
||||||
# ps-sched.sh|grep -v vswitch|awk '($10!=0 && $10!=8 && $3!=2) {if(NR>1)print $2}'
|
|
||||||
cmd_str="ps-sched.sh|grep -v vswitch|awk '("
|
|
||||||
for cpu_num in ${PLATFORM_CPULIST}; do
|
|
||||||
cmd_str=$cmd_str"\$10!="${cpu_num}" && "
|
|
||||||
done
|
|
||||||
cmd_str=$cmd_str"\$3!=2) {if(NR>1)print \$2}'"
|
|
||||||
echo "selection string = $cmd_str"
|
|
||||||
tasklist=($(eval $cmd_str))
|
|
||||||
log_debug "${TAG} There are ${#tasklist[@]} number of tasks to be moved."
|
|
||||||
|
|
||||||
# These sleep tasks are stuck on the wrong core(s). They need to be woken up
|
|
||||||
# so they can be migrated to the right ones. Attaching and detaching strace
|
|
||||||
# momentarily to the task does the trick.
|
|
||||||
for task in ${tasklist[@]}; do
|
|
||||||
strace -p $task 2>/dev/null &
|
|
||||||
pid=$!
|
|
||||||
sleep 0.1
|
|
||||||
kill -SIGINT $pid
|
|
||||||
done
|
|
||||||
tasklist=($(eval $cmd_str))
|
|
||||||
[[ ${#tasklist[@]} -eq 0 ]] && return 0 || return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
# The following function is called by affine-platform.sh to affine tasks to
|
|
||||||
# all available cores during initial startup and subsequent host reboots.
|
|
||||||
################################################################################
|
|
||||||
function affine_tasks_to_all_cores {
|
|
||||||
local pidlist
|
local pidlist
|
||||||
local rc=0
|
|
||||||
|
|
||||||
if [[ "${KERNEL}" == *" RT "* ]]; then
|
pids_excl=$(ps -eL -o pid=,comm= | \
|
||||||
return 0
|
awk -vORS=',' '/eal-intr-thread|kthreadd/ {print $1}' | \
|
||||||
fi
|
sed 's/,$/\n/')
|
||||||
|
pidlist=$(ps --ppid ${pids_excl} -p ${pids_excl} --deselect \
|
||||||
log_debug "${TAG} Affining all tasks to CPU (${FULLSET_CPUS})"
|
-o pid=,cgroup= | \
|
||||||
|
awk '!/k8s-infra|machine.slice/ {print $1; }')
|
||||||
pidlist=$(ps --ppid 2 -p 2 --deselect -o pid= | awk '{ print $1; }')
|
echo "${pidlist[@]}"
|
||||||
for pid in ${pidlist[@]}; do
|
|
||||||
ppid=$(ps -o ppid= -p $pid |tr -d '[:space:]')
|
|
||||||
if [ -z $ppid ] || [ $ppid -eq 2 ]; then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
log_debug "Affining pid $pid, parent pid = $ppid"
|
|
||||||
taskset --all-tasks --pid --cpu-list ${FULLSET_CPUS} $pid &> /dev/null
|
|
||||||
rc=$?
|
|
||||||
[[ $rc -ne 0 ]] && log_error "Failed to set CPU affinity for pid $pid, rc=$rc"
|
|
||||||
done
|
|
||||||
# Write the cpu list to a temp file which will be read and removed when
|
|
||||||
# the tasks are reaffined back to platform cores later on.
|
|
||||||
echo ${FULLSET_CPUS} > ${TASK_AFFINING_INCOMPLETE}
|
|
||||||
|
|
||||||
return $rc
|
|
||||||
}
|
}
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
|
@ -208,35 +132,25 @@ function affine_tasks_to_idle_cores {
|
||||||
|
|
||||||
is_platform_core $cpu
|
is_platform_core $cpu
|
||||||
if [ $? -eq 1 ]; then
|
if [ $? -eq 1 ]; then
|
||||||
# Platform core is added to the idle list by default
|
# Platform core is added to the idle list by default
|
||||||
idle_cpulist=$idle_cpulist$cpu","
|
idle_cpulist=$idle_cpulist$cpu","
|
||||||
else
|
else
|
||||||
# Non platform core is added to the idle list if it is more than 95% idle
|
# Non platform core is added to the idle list if it is more
|
||||||
[[ $(echo "$idle_value > ${IDLE_MARK}"|bc) -eq 1 ]] && idle_cpulist=$idle_cpulist$cpu","
|
# than 95% idle
|
||||||
|
if [[ $(echo "$idle_value > ${IDLE_MARK}"|bc) -eq 1 ]]; then
|
||||||
|
idle_cpulist=$idle_cpulist$cpu","
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
cpu=$(($cpu+1))
|
cpu=$(($cpu+1))
|
||||||
done
|
done
|
||||||
|
|
||||||
idle_cpulist=$(echo $idle_cpulist|sed 's/.$//')
|
idle_cpulist=$(echo $idle_cpulist|sed 's/.$//')
|
||||||
platform_affinity_mask=$(cpulist_to_cpumap ${PLATFORM_CPUS} ${N_CPUS} \
|
|
||||||
|awk '{print tolower($0)}')
|
|
||||||
|
|
||||||
log_debug "${TAG} Affining all tasks to idle CPU ($idle_cpulist)"
|
log_debug "${TAG} Affining all tasks to idle CPU ($idle_cpulist)"
|
||||||
|
pidlist=( $(reaffineable_pids) )
|
||||||
vswitch_pid=$(pgrep vswitch)
|
|
||||||
pidlist=$(ps --ppid 2 -p 2 --deselect -o pid= | awk '{ print $1; }')
|
|
||||||
for pid in ${pidlist[@]}; do
|
for pid in ${pidlist[@]}; do
|
||||||
ppid=$(ps -o ppid= -p $pid |tr -d '[:space:]')
|
taskset --all-tasks --pid --cpu-list \
|
||||||
if [ -z $ppid ] || [ $ppid -eq 2 ] || [ "$pid" = "$vswitch_pid" ]; then
|
${idle_cpulist} ${pid} > /dev/null 2>&1
|
||||||
continue
|
|
||||||
fi
|
|
||||||
pid_affinity_mask=$(taskset -p $pid | awk '{print $6}')
|
|
||||||
if [ "${pid_affinity_mask}" == "${platform_affinity_mask}" ]; then
|
|
||||||
# log_debug "Affining pid $pid to idle cores..."
|
|
||||||
taskset --all-tasks --pid --cpu-list $idle_cpulist $pid &> /dev/null
|
|
||||||
rc=$?
|
|
||||||
[[ $rc -ne 0 ]] && log_error "Failed to set CPU affinity for pid $pid, rc=$rc"
|
|
||||||
fi
|
|
||||||
done
|
done
|
||||||
|
|
||||||
# Save the cpu list to the temp file which will be read and removed when
|
# Save the cpu list to the temp file which will be read and removed when
|
||||||
|
@ -246,10 +160,7 @@ function affine_tasks_to_idle_cores {
|
||||||
}
|
}
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
# The following function is called by either:
|
# The following function is called by sm at the end of swact sequence
|
||||||
# a) nova-compute wrapper script during AIO system initial bringup or reboot
|
|
||||||
# or
|
|
||||||
# b) sm at the end of swact sequence
|
|
||||||
# to re-affine management tasks back to the platform cores.
|
# to re-affine management tasks back to the platform cores.
|
||||||
################################################################################
|
################################################################################
|
||||||
function affine_tasks_to_platform_cores {
|
function affine_tasks_to_platform_cores {
|
||||||
|
@ -259,42 +170,32 @@ function affine_tasks_to_platform_cores {
|
||||||
local count=0
|
local count=0
|
||||||
|
|
||||||
if [ ! -f ${TASK_AFFINING_INCOMPLETE} ]; then
|
if [ ! -f ${TASK_AFFINING_INCOMPLETE} ]; then
|
||||||
dbg_str="${TAG} Either tasks have never been affined to all/idle cores or"
|
dbg_str="${TAG} Either tasks have never been affined to all/idle"
|
||||||
dbg_str=$dbg_str" they have already been reaffined to platform cores."
|
dbg_str="${TAG} cores or they have already been reaffined to"
|
||||||
|
dbg_str="${TAG} platform cores."
|
||||||
log_debug "$dbg_str"
|
log_debug "$dbg_str"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
read cpulist < ${TASK_AFFINING_INCOMPLETE}
|
read cpulist < ${TASK_AFFINING_INCOMPLETE}
|
||||||
affinity_mask=$(cpulist_to_cpumap $cpulist ${N_CPUS}|awk '{print tolower($0)}')
|
|
||||||
|
|
||||||
log_debug "${TAG} Reaffining tasks to platform cores (${PLATFORM_CPUS})..."
|
log_debug "${TAG} Reaffining tasks to platform cores (${PLATFORM_CPUS})..."
|
||||||
pidlist=$(ps --ppid 2 -p 2 --deselect -o pid= | awk '{ print $1; }')
|
pidlist=( $(reaffineable_pids) )
|
||||||
for pid in ${pidlist[@]}; do
|
for pid in ${pidlist[@]}; do
|
||||||
# log_debug "Processing pid $pid..."
|
taskset --all-tasks --pid --cpu-list \
|
||||||
pid_affinity_mask=$(taskset -p $pid | awk '{print $6}')
|
${PLATFORM_CPUS} ${pid} > /dev/null 2>&1
|
||||||
# Only management tasks need to be reaffined. Kernel, vswitch and VM related
|
|
||||||
# tasks were not affined previously so they should have different affinity
|
|
||||||
# mask(s).
|
|
||||||
if [ "${pid_affinity_mask}" == "${affinity_mask}" ]; then
|
|
||||||
count=$(($count+1))
|
|
||||||
# log_debug "Affining pid $pid to platform cores..."
|
|
||||||
taskset --all-tasks --pid --cpu-list ${PLATFORM_CPUS} $pid &> /dev/null
|
|
||||||
rc=$?
|
|
||||||
[[ $rc -ne 0 ]] && log_error "Failed to set CPU affinity for pid $pid, rc=$rc"
|
|
||||||
fi
|
|
||||||
done
|
done
|
||||||
|
|
||||||
# A workaround for lack of "end of swact" state
|
# Reaffine vSwitch tasks that span multiple cpus to platform cpus
|
||||||
fullmask=$(echo ${FULLSET_MASK} | awk '{print tolower($0)}')
|
pidlist=$(ps -eL -o pid=,comm= | awk '/eal-intr-thread/ {print $1}')
|
||||||
if [ "${affinity_mask}" != "${fullmask}" ]; then
|
for pid in ${pidlist[@]}; do
|
||||||
log_debug "${TAG} Schedule an audit and cleanup"
|
grep Cpus_allowed_list /proc/${pid}/task/*/status 2>/dev/null | \
|
||||||
(sleep 60; audit_and_reaffine "0x"$affinity_mask) &
|
sed 's#/# #g' | awk '/,|-/ {print $4}' | \
|
||||||
fi
|
xargs --no-run-if-empty -i{} \
|
||||||
|
taskset --pid --cpu-list ${PLATFORM_CPUS} {} > /dev/null 2>&1
|
||||||
|
done
|
||||||
|
|
||||||
rm -rf ${TASK_AFFINING_INCOMPLETE}
|
rm -rf ${TASK_AFFINING_INCOMPLETE}
|
||||||
log_debug "${TAG} $count tasks were reaffined to platform cores."
|
|
||||||
|
|
||||||
return $rc
|
return $rc
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue