#!/bin/bash
#
# Copyright (c) 2022 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This script does minimal cgroup setup for kubelet. This creates k8s-infra
# cgroup for a minimal set of resource controllers, and configures cpuset
# attributes to span all online cpus and nodes. This will do nothing if
# the k8s-infra cgroup already exists (i.e., assume already configured).
# NOTE: The creation of directories under /sys/fs/cgroup is volatile, and
# does not persist reboots. The cpuset.mems and cpuset.cpus is later updated
# by puppet kubernetes.pp manifest.
#

# Define minimal path
PATH=/bin:/usr/bin:/usr/local/bin

# Log info message to /var/log/daemon.log
function LOG {
    logger -p daemon.info "$0($$): $@"
}

# Log error message to /var/log/daemon.log
function ERROR {
    logger -s -p daemon.error "$0($$): ERROR: $@"
}

# Create minimal cgroup directories and configure cpuset attributes if required
function create_cgroup {
    local cg_name=$1
    local cg_nodeset=$2
    local cg_cpuset=$3

    local CGROUP=/sys/fs/cgroup
    local CONTROLLERS_AUTO_DELETED=("pids" "hugetlb")
    local CONTROLLERS_PRESERVED=("cpuset" "memory" "cpu,cpuacct" "systemd")
    local cnt=''
    local CGDIR=''
    local RC=0

    # Ensure that these cgroups are created every time as they are auto deleted
    for cnt in ${CONTROLLERS_AUTO_DELETED[@]}; do
        CGDIR=${CGROUP}/${cnt}/${cg_name}
        if [ -d ${CGDIR} ]; then
            LOG "Nothing to do, already configured: ${CGDIR}."
            continue
        fi
        LOG "Creating: ${CGDIR}"
        mkdir -p ${CGDIR}
        RC=$?
        if [ ${RC} -ne 0 ]; then
            ERROR "Creating: ${CGDIR}, rc=${RC}"
            exit ${RC}
        fi
    done

    # These cgroups are preserved so if any of these are encountered additional
    # cgroup setup is not required
    for cnt in ${CONTROLLERS_PRESERVED[@]}; do
        CGDIR=${CGROUP}/${cnt}/${cg_name}
        if [ -d ${CGDIR} ]; then
            LOG "Nothing to do, already configured: ${CGDIR}."
            exit ${RC}
        fi
        LOG "Creating: ${CGDIR}"
        mkdir -p ${CGDIR}
        RC=$?
        if [ ${RC} -ne 0 ]; then
            ERROR "Creating: ${CGDIR}, rc=${RC}"
            exit ${RC}
        fi
    done

    # Customize cpuset attributes
    LOG "Configuring cgroup: ${cg_name}, nodeset: ${cg_nodeset}, cpuset: ${cg_cpuset}"
    CGDIR=${CGROUP}/cpuset/${cg_name}
    local CGMEMS=${CGDIR}/cpuset.mems
    local CGCPUS=${CGDIR}/cpuset.cpus
    local CGTASKS=${CGDIR}/tasks

    # Assign cgroup memory nodeset
    LOG "Assign nodeset ${cg_nodeset} to ${CGMEMS}"
    /bin/echo ${cg_nodeset} > ${CGMEMS}
    RC=$?
    if [ ${RC} -ne 0 ]; then
        ERROR "Unable to write to: ${CGMEMS}, rc=${RC}"
        exit ${RC}
    fi

    # Assign cgroup cpus
    LOG "Assign cpuset ${cg_cpuset} to ${CGCPUS}"
    /bin/echo ${cg_cpuset} > ${CGCPUS}
    RC=$?
    if [ ${RC} -ne 0 ]; then
        ERROR "Assigning: ${cg_cpuset} to ${CGCPUS}, rc=${RC}"
        exit ${RC}
    fi

    # Set file ownership
    chown root:root ${CGMEMS} ${CGCPUS} ${CGTASKS}
    RC=$?
    if [ ${RC} -ne 0 ]; then
        ERROR "Setting owner for: ${CGMEMS}, ${CGCPUS}, ${CGTASKS}, rc=${RC}"
        exit ${RC}
    fi

    # Set file mode permissions
    chmod 644 ${CGMEMS} ${CGCPUS} ${CGTASKS}
    RC=$?
    if [ ${RC} -ne 0 ]; then
        ERROR "Setting mode for: ${CGMEMS}, ${CGCPUS}, ${CGTASKS}, rc=${RC}"
        exit ${RC}
    fi

    return ${RC}
}

if [ ${UID} -ne 0 ]; then
    ERROR "Require sudo/root."
    exit 1
fi

# Configure default kubepods cpuset to span all online cpus and nodes.
ONLINE_NODESET=$(/bin/cat /sys/devices/system/node/online)
ONLINE_CPUSET=$(/bin/cat /sys/devices/system/cpu/online)

# Configure kubelet cgroup to match cgroupRoot.
create_cgroup 'k8s-infra' ${ONLINE_NODESET} ${ONLINE_CPUSET}

exit $?