diff --git a/utilities/worker-utils/.gitignore b/utilities/worker-utils/.gitignore new file mode 100644 index 00000000..06c1a0e8 --- /dev/null +++ b/utilities/worker-utils/.gitignore @@ -0,0 +1,6 @@ +!.distro +.distro/centos7/rpmbuild/RPMS +.distro/centos7/rpmbuild/SRPMS +.distro/centos7/rpmbuild/BUILD +.distro/centos7/rpmbuild/BUILDROOT +.distro/centos7/rpmbuild/SOURCES/worker-utils*tar.gz diff --git a/utilities/worker-utils/centos/build_srpm.data b/utilities/worker-utils/centos/build_srpm.data new file mode 100644 index 00000000..8a107e97 --- /dev/null +++ b/utilities/worker-utils/centos/build_srpm.data @@ -0,0 +1,3 @@ +SRC_DIR="worker-utils" +COPY_LIST="$SRC_DIR/LICENSE" +TIS_PATCH_VER=1 diff --git a/utilities/worker-utils/centos/worker-utils.spec b/utilities/worker-utils/centos/worker-utils.spec new file mode 100644 index 00000000..560b41e3 --- /dev/null +++ b/utilities/worker-utils/centos/worker-utils.spec @@ -0,0 +1,55 @@ +Summary: Initial worker node resource reservation and misc. utilities +Name: worker-utils +Version: 1.0 +Release: %{tis_patch_ver}%{?_tis_dist} +License: Apache-2.0 +Group: base +Packager: Wind River +URL: unknown +Source0: %{name}-%{version}.tar.gz +Source1: LICENSE + +BuildRequires: systemd-devel +Requires: systemd +Requires: python +Requires: /bin/systemctl + +%description +Initial worker node resource reservation and misc. utilities + +%define local_bindir /usr/bin/ +%define local_etc_initd /etc/init.d/ +%define local_etc_platform /etc/platform/ +%define local_etc_goenabledd /etc/goenabled.d/ + +%define debug_package %{nil} + +%prep +%setup + +%build +make + +%install +make install BINDIR=%{buildroot}%{local_bindir} \ + INITDDIR=%{buildroot}%{local_etc_initd} \ + GOENABLEDDIR=%{buildroot}%{local_etc_goenabledd} \ + PLATFORMCONFDIR=%{buildroot}%{local_etc_platform} \ + SYSTEMDDIR=%{buildroot}%{_unitdir} + +%post +/bin/systemctl enable affine-platform.sh.service >/dev/null 2>&1 + +%clean +rm -rf $RPM_BUILD_ROOT + +%files + +%defattr(-,root,root,-) + +%{local_bindir}/* +%{local_etc_initd}/* +%{local_etc_goenabledd}/* +%config(noreplace) %{local_etc_platform}/worker_reserved.conf + +%{_unitdir}/affine-platform.sh.service diff --git a/utilities/worker-utils/worker-utils/LICENSE b/utilities/worker-utils/worker-utils/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/utilities/worker-utils/worker-utils/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/utilities/worker-utils/worker-utils/Makefile b/utilities/worker-utils/worker-utils/Makefile new file mode 100644 index 00000000..97ab6c60 --- /dev/null +++ b/utilities/worker-utils/worker-utils/Makefile @@ -0,0 +1,31 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# + +BINDIR ?= /usr/bin +INITDDIR ?= /etc/init.d/ +GOENABLEDDIR ?= /etc/goenabled.d/ +PLATFORMCONFDIR ?= /etc/platform +SYSTEMDDIR ?= /usr/lib/systemd/system/ + +all: + python -m compileall topology.py + +install: + install -d -m 755 $(BINDIR) + install -d -m 755 $(INITDDIR) + install -d -m 755 $(GOENABLEDDIR) + install -d -m 755 $(PLATFORMCONFDIR) + install -d -m 755 $(SYSTEMDDIR) + install -p -D -m 755 affine-platform.sh $(INITDDIR)/affine-platform.sh + install -p -D -m 755 cpumap_functions.sh $(INITDDIR)/cpumap_functions.sh + install -p -D -m 755 task_affinity_functions.sh $(INITDDIR)/task_affinity_functions.sh + install -p -D -m 755 ps-sched.sh $(BINDIR)/ps-sched.sh + install -p -D -m 755 topology.py $(BINDIR)/topology.py + install -p -D -m 755 topology.pyc $(BINDIR)/topology.pyc + install -p -D -m 755 affine-interrupts.sh $(BINDIR)/affine-interrupts.sh + install -p -D -m 755 set-cpu-wakeup-latency.sh $(BINDIR)/set-cpu-wakeup-latency.sh + install -p -D -m 755 topology $(BINDIR)/topology + install -p -D -m 755 worker_reserved.conf $(PLATFORMCONFDIR)/worker_reserved.conf + install -p -D -m 755 worker-goenabled.sh $(GOENABLEDDIR)/worker-goenabled.sh + install -p -D -m 664 affine-platform.sh.service $(SYSTEMDDIR)/affine-platform.sh.service diff --git a/utilities/worker-utils/worker-utils/affine-interrupts.sh b/utilities/worker-utils/worker-utils/affine-interrupts.sh new file mode 100644 index 00000000..6a7c228e --- /dev/null +++ b/utilities/worker-utils/worker-utils/affine-interrupts.sh @@ -0,0 +1,59 @@ +#!/bin/bash +################################################################################ +# Copyright (c) 2015-2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +################################################################################ +# +# Purpose: +# Affine the interface IRQ to specified cpulist. +# +# Usage: /usr/bin/affine-interrupts.sh interface cpulist +# +# Define minimal path +PATH=/bin:/usr/bin:/usr/local/bin + +# logger setup +WHOAMI=`basename $0` +LOG_FACILITY=user +LOG_PRIORITY=info +TMPLOG=/tmp/${WHOAMI}.log + +# LOG() - generates log and puts in temporary file +function LOG { + logger -t "${0##*/}[$$]" -p ${LOG_FACILITY}.${LOG_PRIORITY} "$@" + echo "${0##*/}[$$]" "$@" >> ${TMPLOG} +} +function INFO { + MSG="INFO" + LOG "${MSG} $@" +} +function ERROR { + MSG="ERROR" + LOG "${MSG} $@" +} + +if [ "$#" -ne 2 ]; then + ERROR "Interface name and cpulist are required" + exit 1 +fi + +interface=$1 +cpulist=$2 + +# Find PCI device matching interface, keep last matching device name +dev=$(find /sys/devices -name "${interface}" | \ + perl -ne 'print $1 if /([[:xdigit:]]{4}:[[:xdigit:]]{2}:[[:xdigit:]]{2}\.[[:xdigit:]])\/[[:alpha:]]/;') + +# Obtain all IRQs for this device +irq=$(cat /sys/bus/pci/devices/${dev}/irq 2>/dev/null) +msi_irqs=$(ls /sys/bus/pci/devices/${dev}/msi_irqs 2>/dev/null | xargs) + +INFO $LINENO "affine ${interface} (dev:${dev} irq:${irq} msi_irqs:${msi_irqs}) with cpus (${cpulist})" + +for i in $(echo "${irq} ${msi_irqs}"); do echo $i; done | \ + xargs --no-run-if-empty -i{} \ + /bin/bash -c "[[ -e /proc/irq/{} ]] && echo ${cpulist} > /proc/irq/{}/smp_affinity_list" 2>/dev/null + +exit 0 diff --git a/utilities/worker-utils/worker-utils/affine-platform.sh b/utilities/worker-utils/worker-utils/affine-platform.sh new file mode 100755 index 00000000..13f46163 --- /dev/null +++ b/utilities/worker-utils/worker-utils/affine-platform.sh @@ -0,0 +1,163 @@ +#!/bin/bash +################################################################################ +# Copyright (c) 2013 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +################################################################################ +# Define minimal path +PATH=/bin:/usr/bin:/usr/local/bin + +LOG_FUNCTIONS=${LOG_FUNCTIONS:-"/etc/init.d/log_functions.sh"} +CPUMAP_FUNCTIONS=${CPUMAP_FUNCTIONS:-"/etc/init.d/cpumap_functions.sh"} +TASK_AFFINITY_FUNCTIONS=${TASK_AFFINITY_FUNCTIONS:-"/etc/init.d/task_affinity_functions.sh"} +source /etc/init.d/functions +[[ -e ${LOG_FUNCTIONS} ]] && source ${LOG_FUNCTIONS} +[[ -e ${CPUMAP_FUNCTIONS} ]] && source ${CPUMAP_FUNCTIONS} +[[ -e ${TASK_AFFINITY_FUNCTIONS} ]] && source ${TASK_AFFINITY_FUNCTIONS} +linkname=$(readlink -n -f $0) +scriptname=$(basename $linkname) + +# Enable debug logs +LOG_DEBUG=1 + +. /etc/platform/platform.conf + +################################################################################ +# Affine all running tasks to the CPULIST provided in the first parameter. +################################################################################ +function affine_tasks { + local CPULIST=$1 + local PIDLIST + local RET=0 + + # Affine non-kernel-thread tasks (excluded [kthreadd] and its children) to all available + # cores. They will be reaffined to platform cores later on as part of nova-compute + # launch. + log_debug "Affining all tasks to all available CPUs..." + affine_tasks_to_all_cores + RET=$? + if [ $RET -ne 0 ]; then + log_error "Some tasks failed to be affined to all cores." + fi + + # Get number of logical cpus + N_CPUS=$(cat /proc/cpuinfo 2>/dev/null | \ + awk '/^[pP]rocessor/ { n +=1 } END { print (n>0) ? n : 1}') + + # Calculate platform cores cpumap + PLATFORM_COREMASK=$(cpulist_to_cpumap ${CPULIST} ${N_CPUS}) + + # Set default IRQ affinity + echo ${PLATFORM_COREMASK} > /proc/irq/default_smp_affinity + + # Affine all PCI/MSI interrupts to platform cores; this overrides + # irqaffinity boot arg, since that does not handle IRQs for PCI devices + # on numa nodes that do not intersect with platform cores. + PCIDEVS=/sys/bus/pci/devices + declare -a irqs=() + irqs+=($(cat ${PCIDEVS}/*/irq 2>/dev/null | xargs)) + irqs+=($(ls ${PCIDEVS}/*/msi_irqs 2>/dev/null | grep -E '^[0-9]+$' | xargs)) + # flatten list of irqs, removing duplicates + irqs=($(echo ${irqs[@]} | tr ' ' '\n' | sort -nu)) + log_debug "Affining all PCI/MSI irqs(${irqs[@]}) with cpus (${CPULIST})" + for i in ${irqs[@]}; do + /bin/bash -c "[[ -e /proc/irq/${i} ]] && echo ${CPULIST} > /proc/irq/${i}/smp_affinity_list" 2>/dev/null + done + if [[ "$subfunction" == *"worker,lowlatency" ]]; then + # Affine work queues to platform cores + echo ${PLATFORM_COREMASK} > /sys/devices/virtual/workqueue/cpumask + echo ${PLATFORM_COREMASK} > /sys/bus/workqueue/devices/writeback/cpumask + + # On low latency compute reassign the per cpu threads rcuc, ksoftirq, + # ktimersoftd to FIFO along with the specified priority + PIDLIST=$( ps -e -p 2 |grep rcuc | awk '{ print $1; }') + for PID in ${PIDLIST[@]}; do + chrt -p -f 4 ${PID} 2>/dev/null + done + + PIDLIST=$( ps -e -p 2 |grep ksoftirq | awk '{ print $1; }') + for PID in ${PIDLIST[@]}; do + chrt -p -f 2 ${PID} 2>/dev/null + done + + PIDLIST=$( ps -e -p 2 |grep ktimersoftd | awk '{ print $1; }') + for PID in ${PIDLIST[@]}; do + chrt -p -f 3 ${PID} 2>/dev/null + done + fi + + return 0 +} + +################################################################################ +# Start Action +################################################################################ +function start { + local RET=0 + + echo -n "Starting ${scriptname}: " + + ## Check whether we are root (need root for taskset) + if [ $UID -ne 0 ]; then + log_error "require root or sudo" + RET=1 + return ${RET} + fi + + ## Define platform cpulist to be thread siblings of core 0 + PLATFORM_CPULIST=$(get_platform_cpu_list) + + # Affine all tasks to platform cpulist + affine_tasks ${PLATFORM_CPULIST} + RET=$? + if [ ${RET} -ne 0 ]; then + log_error "Failed to affine tasks ${PLATFORM_CPULIST}, rc=${RET}" + return ${RET} + fi + + print_status ${RET} + return ${RET} +} + +################################################################################ +# Stop Action - don't do anything +################################################################################ +function stop { + local RET=0 + echo -n "Stopping ${scriptname}: " + print_status ${RET} + return ${RET} +} + +################################################################################ +# Restart Action +################################################################################ +function restart { + stop + start +} + +################################################################################ +# Main Entry +# +################################################################################ +case "$1" in +start) + start + ;; +stop) + stop + ;; +restart|reload) + restart + ;; +status) + echo -n "OK" + ;; +*) + echo $"Usage: $0 {start|stop|restart|reload|status}" + exit 1 +esac + +exit $? diff --git a/utilities/worker-utils/worker-utils/affine-platform.sh.service b/utilities/worker-utils/worker-utils/affine-platform.sh.service new file mode 100644 index 00000000..f124182b --- /dev/null +++ b/utilities/worker-utils/worker-utils/affine-platform.sh.service @@ -0,0 +1,14 @@ +[Unit] +Description=Titanium Cloud Affine Platform +After=syslog.service network.service dbus.service sw-patch.service +Before=workerconfig.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/etc/init.d/affine-platform.sh start +ExecStop=/etc/init.d/affine-platform.sh stop +ExecReload=/etc/init.d/affine-platform.sh restart + +[Install] +WantedBy=multi-user.target diff --git a/utilities/worker-utils/worker-utils/cpumap_functions.sh b/utilities/worker-utils/worker-utils/cpumap_functions.sh new file mode 100644 index 00000000..3202fc3b --- /dev/null +++ b/utilities/worker-utils/worker-utils/cpumap_functions.sh @@ -0,0 +1,383 @@ +#!/bin/bash +################################################################################ +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +################################################################################ + +source /etc/platform/platform.conf + +################################################################################ +# Utility function to expand a sequence of numbers (e.g., 0-7,16-23) +################################################################################ +function expand_sequence { + SEQUENCE=(${1//,/ }) + DELIMITER=${2:-","} + + LIST= + for entry in ${SEQUENCE[@]}; do + range=(${entry/-/ }) + a=${range[0]} + b=${range[1]:-${range[0]}} + + for i in $(seq $a $b); do + LIST="${LIST}${DELIMITER}${i}" + done + done + echo ${LIST:1} +} + +################################################################################ +# Append a string to comma separated list string +################################################################################ +function append_list { + local PUSH=$1 + local LIST=$2 + if [ -z "${LIST}" ]; then + LIST=${PUSH} + else + LIST="${LIST},${PUSH}" + fi + echo ${LIST} + return 0 +} + +################################################################################ +# Condense a sequence of numbers to a list of ranges (e.g, 7-12,15-16) +################################################################################ +function condense_sequence { + local arr=( $(printf '%s\n' "$@" | sort -n) ) + local first + local last + local cpulist="" + for ((i=0; i < ${#arr[@]}; i++)); do + num=${arr[$i]} + if [[ -z $first ]]; then + first=$num + last=$num + continue + fi + if [[ num -ne $((last + 1)) ]]; then + if [[ first -eq last ]]; then + cpulist=$(append_list ${first} ${cpulist}) + else + cpulist=$(append_list "${first}-${last}" ${cpulist}) + fi + first=$num + last=$num + else + : $((last++)) + fi + done + if [[ first -eq last ]]; then + cpulist=$(append_list ${first} ${cpulist}) + else + cpulist=$(append_list "${first}-${last}" ${cpulist}) + fi + echo "$cpulist" +} + +################################################################################ +# Converts a CPULIST (e.g., 0-7,16-23) to a CPUMAP (e.g., 0x00FF00FF). The +# CPU map is returned as a string representation of a large hexidecimal +# number but without the leading "0x" characters. +# +################################################################################ +function cpulist_to_cpumap { + local CPULIST=$1 + local NR_CPUS=$2 + local CPUMAP=0 + local CPUID=0 + if [ -z "${NR_CPUS}" ] || [ ${NR_CPUS} -eq 0 ]; then + echo 0 + return 0 + fi + for CPUID in $(expand_sequence $CPULIST " "); do + if [ "${CPUID}" -lt "${NR_CPUS}" ]; then + CPUMAP=$(echo "${CPUMAP} + (2^${CPUID})" | bc -l) + fi + done + + echo "obase=16;ibase=10;${CPUMAP}" | bc -l + return 0 +} + +################################################################################ +# Converts a CPUMAP (e.g., 0x00FF00FF) to a CPULIST (e.g., 0-7,16-23). The +# CPUMAP is expected in hexidecimal (base=10) form without the leading "0x" +# characters. +# +################################################################################ +function cpumap_to_cpulist { + local CPUMAP + CPUMAP=$(echo "obase=10;ibase=16;$1" | bc -l) + local NR_CPUS=$2 + local list=() + local cpulist="" + for((i=0; i < NR_CPUS; i++)) + do + ## Since 'bc' does not support any bitwise operators this expression: + ## if (CPUMAP & (1 << CPUID)) + ## has to be rewritten like this: + ## if (CPUMAP % (2**(CPUID+1)) > ((2**(CPUID)) - 1)) + ## + ISSET=$(echo "scale=0; (${CPUMAP} % 2^(${i}+1)) > (2^${i})-1" | bc -l) + if [ "${ISSET}" -ne 0 ]; then + list+=($i) + fi + done + cpulist=$(condense_sequence ${list[@]} ) + echo "$cpulist" + return 0 +} + +################################################################################ +# Bitwise NOT of a hexidecimal representation of a CPULIST. The value is +# returned as a hexidecimal value but without the leading "0x" characters +# +################################################################################ +function invert_cpumap { + local CPUMAP + CPUMAP=$(echo "obase=10;ibase=16;$1" | bc -l) + local NR_CPUS=$2 + local INVERSE_CPUMAP=0 + + for CPUID in $(seq 0 $((NR_CPUS - 1))); do + ## See comment in previous function + ISSET=$(echo "scale=0; (${CPUMAP} % 2^(${CPUID}+1)) > (2^${CPUID})-1" | bc -l) + if [ "${ISSET}" -eq 1 ]; then + continue + fi + + INVERSE_CPUMAP=$(echo "${INVERSE_CPUMAP} + (2^${CPUID})" | bc -l) + done + + echo "obase=16;ibase=10;${INVERSE_CPUMAP}" | bc -l + return 0 +} + +################################################################################ +# Builds the complement representation of a CPULIST +# +################################################################################ +function invert_cpulist { + local CPULIST=$1 + local NR_CPUS=$2 + local CPUMAP + CPUMAP=$(cpulist_to_cpumap ${CPULIST} ${NR_CPUS}) + cpumap_to_cpulist $(invert_cpumap ${CPUMAP} ${NR_CPUS}) ${NR_CPUS} + return 0 +} + +################################################################################ +# in_list() - check whether item is contained in list +# param: item +# param: list (i.e. 0-3,8-11) +# returns: 0 - item is contained in list; +# 1 - item is not contained in list +# +################################################################################ +function in_list { + local item="$1" + local list="$2" + + # expand list format 0-3,8-11 to a full sequence {0..3} {8..11} + local exp_list + exp_list=$(echo ${list} | \ + sed -e 's#,# #g' -e 's#\([0-9]*\)-\([0-9]*\)#{\1\.\.\2}#g') + + local e + for e in $(eval echo ${exp_list}); do + [[ "$e" == "$item" ]] && return 0 + done + return 1 +} + +################################################################################ +# any_in_list() - check if any item of sublist is contained in list +# param: sublist +# param: list +# returns: 0 - an item of sublist is contained in list; +# 1 - no sublist items contained in list +# +################################################################################ +function any_in_list { + local sublist="$1" + local list="$2" + local e + local exp_list + + # expand list format 0-3,8-11 to a full sequence {0..3} {8..11} + exp_list=$(echo ${list} | \ + sed -e 's#,# #g' -e 's#\([0-9]*\)-\([0-9]*\)#{\1\.\.\2}#g') + declare -A a_list + for e in $(eval echo ${exp_list}); do + a_list[$e]=1 + done + + # expand list format 0-3,8-11 to a full sequence {0..3} {8..11} + exp_list=$(echo ${sublist} | \ + sed -e 's#,# #g' -e 's#\([0-9]*\)-\([0-9]*\)#{\1\.\.\2}#g') + declare -A a_sublist + for e in $(eval echo ${exp_list}); do + a_sublist[$e]=1 + done + + # Check if any element of sublist is in list + for e in "${!a_sublist[@]}"; do + if [[ "${a_list[$e]}" == 1 ]]; then + return 0 # matches + fi + done + return 1 # no match +} + +################################################################################ +# Return list of CPUs reserved for platform +################################################################################ +function get_platform_cpu_list { + ## Define platform cpulist based on engineering a number of cores and + ## whether this is a combo or not, and include SMT siblings. + if [[ $subfunction = *worker* ]]; then + RESERVE_CONF="/etc/platform/worker_reserved.conf" + [[ -e ${RESERVE_CONF} ]] && source ${RESERVE_CONF} + if [ -n "$PLATFORM_CPU_LIST" ];then + echo "$PLATFORM_CPU_LIST" + return 0 + fi + fi + + local PLATFORM_SOCKET=0 + local PLATFORM_START=0 + local PLATFORM_CORES=1 + if [ "$nodetype" = "controller" ]; then + PLATFORM_CORES=$(($PLATFORM_CORES+1)) + fi + local PLATFORM_CPULIST + PLATFORM_CPULIST=$(topology_to_cpulist ${PLATFORM_SOCKET} ${PLATFORM_START} ${PLATFORM_CORES}) + echo ${PLATFORM_CPULIST} +} + +################################################################################ +# Return list of CPUs reserved for vswitch +################################################################################ +function get_vswitch_cpu_list { + ## Define default avp cpulist based on engineered number of platform cores, + ## engineered avp cores, and include SMT siblings. + if [[ $subfunction = *worker* ]]; then + VSWITCH_CONF="/etc/vswitch/vswitch.conf" + [[ -e ${VSWITCH_CONF} ]] && source ${VSWITCH_CONF} + if [ -n "$VSWITCH_CPU_LIST" ];then + echo "$VSWITCH_CPU_LIST" + return 0 + fi + fi + + local N_CORES_IN_PKG + N_CORES_IN_PKG=$(cat /proc/cpuinfo 2>/dev/null | \ + awk '/^cpu cores/ {n = $4} END { print (n>0) ? n : 1 }') + # engineer platform cores + local PLATFORM_CORES=1 + if [ "$nodetype" = "controller" ]; then + PLATFORM_CORES=$(($PLATFORM_CORES+1)) + fi + + # engineer AVP cores + local AVP_SOCKET=0 + local AVP_START=${PLATFORM_CORES} + local AVP_CORES=1 + if [ ${N_CORES_IN_PKG} -gt 4 ]; then + AVP_CORES=$(($AVP_CORES+1)) + fi + local AVP_CPULIST + AVP_CPULIST=$(topology_to_cpulist ${AVP_SOCKET} ${AVP_START} ${AVP_CORES}) + echo ${AVP_CPULIST} +} + +################################################################################ +# vswitch_expanded_cpu_list() - compute the vswitch cpu list, including it's siblings +################################################################################ +function vswitch_expanded_cpu_list { + list=$(get_vswitch_cpu_list) + + # Expand vswitch cpulist + vswitch_cpulist=$(expand_sequence ${list} " ") + + cpulist="" + for e in $vswitch_cpulist; do + # claim hyperthread siblings if SMT enabled + SIBLINGS_CPULIST=$(cat /sys/devices/system/cpu/cpu${e}/topology/thread_siblings_list 2>/dev/null) + siblings_cpulist=$(expand_sequence ${SIBLINGS_CPULIST} " ") + for s in $siblings_cpulist; do + in_list ${s} ${cpulist} + if [ $? -eq 1 ]; then + cpulist=$(append_list ${s} ${cpulist}) + fi + done + done + + echo "$cpulist" + return 0 +} + +################################################################################ +# platform_expanded_cpu_list() - compute the platform cpu list, including it's siblings +################################################################################ +function platform_expanded_cpu_list { + list=$(get_platform_cpu_list) + + # Expand platform cpulist + platform_cpulist=$(expand_sequence ${list} " ") + + cpulist="" + for e in $platform_cpulist; do + # claim hyperthread siblings if SMT enabled + SIBLINGS_CPULIST=$(cat /sys/devices/system/cpu/cpu${e}/topology/thread_siblings_list 2>/dev/null) + siblings_cpulist=$(expand_sequence ${SIBLINGS_CPULIST} " ") + for s in $siblings_cpulist; do + in_list ${s} ${cpulist} + if [ $? -eq 1 ]; then + cpulist=$(append_list ${s} ${cpulist}) + fi + done + done + + echo "$cpulist" + return 0 +} + +################################################################################ +# Return list of CPUs based on cpu topology. Select the socket, starting core +# within the socket, select number of cores, and SMT siblings. +################################################################################ +function topology_to_cpulist { + local SOCKET=$1 + local CORE_START=$2 + local NUM_CORES=$3 + local CPULIST + CPULIST=$(cat /proc/cpuinfo 2>/dev/null | perl -sne \ +'BEGIN { %T = {}; %H = {}; $L = $P = $C = $S = 0; } +{ + if (/processor\s+:\s+(\d+)/) { $L = $1; } + if (/physical id\s+:\s+(\d+)/) { $P = $1; } + if (/core id\s+:\s+(\d+)/) { + $C = $1; + $T{$P}{$C}++; + $S = $T{$P}{$C}; + $H{$P}{$C}{$S} = $L; + } +} +END { + @cores = sort { $a <=> $b } keys $T{$socket}; + @sel_cores = splice @cores, $core_start, $num_cores; + @lcpus = (); + for $C (@sel_cores) { + for $S (sort {$a <=> $b } keys %{ $H{$socket}{$C} }) { + push @lcpus, $H{$socket}{$C}{$S}; + } + } + printf "%s\n", join(",", @lcpus); +}' -- -socket=${SOCKET} -core_start=${CORE_START} -num_cores=${NUM_CORES}) + echo ${CPULIST} +} diff --git a/utilities/worker-utils/worker-utils/cpumap_functions_unit_test.sh b/utilities/worker-utils/worker-utils/cpumap_functions_unit_test.sh new file mode 100644 index 00000000..941352df --- /dev/null +++ b/utilities/worker-utils/worker-utils/cpumap_functions_unit_test.sh @@ -0,0 +1,241 @@ +#!/bin/bash + +# +# Copyright (c) 2015-2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +source /etc/init.d/cpumap_functions.sh + +export NR_CPUS_LIST=("4" "8" "16" "32" "64" "128") +if [ ! -z ${1} ]; then + NR_CPUS_LIST=(${1//,/ }) +fi + +function test_cpumap_to_cpulist { + local NR_CPUS=$1 + declare -A CPULISTS + + if [ ${NR_CPUS} -ge 4 ]; then + CPULISTS["0"]="" + CPULISTS["1"]="0" + CPULISTS["2"]="1" + CPULISTS["3"]="0-1" + CPULISTS["5"]="0,2" + CPULISTS["7"]="0-2" + CPULISTS["F"]="0-3" + CPULISTS["9"]="0,3" + fi + if [ ${NR_CPUS} -ge 8 ]; then + CPULISTS["00"]="" + CPULISTS["11"]="0,4" + CPULISTS["FF"]="0-7" + CPULISTS["81"]="0,7" + fi + if [ ${NR_CPUS} -ge 16 ]; then + CPULISTS["0000"]="" + CPULISTS["1111"]="0,4,8,12" + CPULISTS["FFF"]="0-11" + CPULISTS["F0F"]="0-3,8-11" + CPULISTS["F0F0"]="4-7,12-15" + CPULISTS["FFFF"]="0-15" + CPULISTS["FFFE"]="1-15" + CPULISTS["8001"]="0,15" + fi + if [ ${NR_CPUS} -ge 32 ]; then + CPULISTS["00000000"]="" + CPULISTS["11111111"]="0,4,8,12,16,20,24,28" + CPULISTS["0F0F0F0F"]="0-3,8-11,16-19,24-27" + CPULISTS["F0F0F0F0"]="4-7,12-15,20-23,28-31" + CPULISTS["FFFFFFFF"]="0-31" + CPULISTS["FFFFFFFE"]="1-31" + CPULISTS["80000001"]="0,31" + fi + if [ ${NR_CPUS} -ge 64 ]; then + CPULISTS["0000000000000000"]="" + CPULISTS["1111111111111111"]="0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60" + CPULISTS["0F0F0F0F0F0F0F0F"]="0-3,8-11,16-19,24-27,32-35,40-43,48-51,56-59" + CPULISTS["F0F0F0F0F0F0F0F0"]="4-7,12-15,20-23,28-31,36-39,44-47,52-55,60-63" + CPULISTS["FFFFFFFFFFFFFFFF"]="0-63" + CPULISTS["FFFFFFFFFFFFFFFE"]="1-63" + CPULISTS["8000000000000001"]="0,63" + fi + if [ ${NR_CPUS} -ge 128 ]; then + CPULISTS["00000000000000000000000000000000"]="" + CPULISTS["11111111111111111111111111111111"]="0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124" + CPULISTS["0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F"]="0-3,8-11,16-19,24-27,32-35,40-43,48-51,56-59,64-67,72-75,80-83,88-91,96-99,104-107,112-115,120-123" + CPULISTS["F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0"]="4-7,12-15,20-23,28-31,36-39,44-47,52-55,60-63,68-71,76-79,84-87,92-95,100-103,108-111,116-119,124-127" + CPULISTS["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"]="0-127" + CPULISTS["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"]="1-127" + CPULISTS["80000000000000000000000000000001"]="0,127" + fi + + for CPUMAP in ${!CPULISTS[@]}; do + EXPECTED=${CPULISTS[${CPUMAP}]} + CPULIST=$(cpumap_to_cpulist ${CPUMAP} ${NR_CPUS}) + if [ "${CPULIST}" != "${EXPECTED}" ]; then + printf "\n" + echo "error: (cpumap_to_list ${CPUMAP} ${NR_CPUS}) returned \"${CPULIST}\" instead of \"${EXPECTED}\"" + fi + printf "." + done + + printf "\n" +} + +function test_cpulist_to_cpumap { + local NR_CPUS=$1 + declare -A CPUMAPS + + if [ ${NR_CPUS} -ge 4 ]; then + CPUMAPS[" "]="0" + CPUMAPS["0"]="1" + CPUMAPS["1"]="2" + CPUMAPS["0-1"]="3" + CPUMAPS["0,2"]="5" + CPUMAPS["0-2"]="7" + CPUMAPS["0-3"]="F" + CPUMAPS["0,3"]="9" + fi + if [ ${NR_CPUS} -ge 8 ]; then + CPUMAPS["0,4"]="11" + CPUMAPS["0-7"]="FF" + CPUMAPS["0,7"]="81" + fi + if [ ${NR_CPUS} -ge 16 ]; then + CPUMAPS["0,4,8,12"]="1111" + CPUMAPS["0-11"]="FFF" + CPUMAPS["0-3,8-11"]="F0F" + CPUMAPS["4-7,12-15"]="F0F0" + CPUMAPS["0-15"]="FFFF" + CPUMAPS["1-15"]="FFFE" + CPUMAPS["0,15"]="8001" + fi + if [ ${NR_CPUS} -ge 32 ]; then + CPUMAPS["0,4,8,12,16,20,24,28"]="11111111" + CPUMAPS["0-3,8-11,16-19,24-27"]="F0F0F0F" + CPUMAPS["4-7,12-15,20-23,28-31"]="F0F0F0F0" + CPUMAPS["0-31"]="FFFFFFFF" + CPUMAPS["1-31"]="FFFFFFFE" + CPUMAPS["0,31"]="80000001" + fi + if [ ${NR_CPUS} -ge 64 ]; then + CPUMAPS["0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60"]="1111111111111111" + CPUMAPS["0-3,8-11,16-19,24-27,32-35,40-43,48-51,56-59"]="F0F0F0F0F0F0F0F" + CPUMAPS["4-7,12-15,20-23,28-31,36-39,44-47,52-55,60-63"]="F0F0F0F0F0F0F0F0" + CPUMAPS["0-63"]="FFFFFFFFFFFFFFFF" + CPUMAPS["1-63"]="FFFFFFFFFFFFFFFE" + CPUMAPS["0,63"]="8000000000000001" + fi + if [ ${NR_CPUS} -ge 128 ]; then + CPUMAPS["0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124"]="11111111111111111111111111111111" + CPUMAPS["0-3,8-11,16-19,24-27,32-35,40-43,48-51,56-59,64-67,72-75,80-83,88-91,96-99,104-107,112-115,120-123"]="F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F" + CPUMAPS["4-7,12-15,20-23,28-31,36-39,44-47,52-55,60-63,68-71,76-79,84-87,92-95,100-103,108-111,116-119,124-127"]="F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0" + CPUMAPS["0-127"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" + CPUMAPS["1-127"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE" + CPUMAPS["0,127"]="80000000000000000000000000000001" + fi + + for CPULIST in ${!CPUMAPS[@]}; do + EXPECTED=${CPUMAPS[${CPULIST}]} + CPUMAP=$(cpulist_to_cpumap ${CPULIST} ${NR_CPUS}) + if [ "${CPUMAP}" != "${EXPECTED}" ]; then + printf "\n" + echo "error: (cpulist_to_cpumap ${CPULIST} ${NR_CPUS}) returned \"${CPUMAP}\" instead of \"${EXPECTED}\"" + fi + printf "." + done + + printf "\n" +} + +function test_invert_cpumap { + local NR_CPUS=$1 + declare -A INVERSES + + if [ $((${NR_CPUS} % 4)) -ne 0 ]; then + echo "test_invert_cpumap skipping NR_CPUS=${NR_CPUS}; not a multiple of 4" + return 0 + fi + + if [ ${NR_CPUS} -ge 4 ]; then + INVERSES["0"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" + INVERSES["1"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE" + INVERSES["2"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD" + INVERSES["3"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFC" + INVERSES["5"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFA" + INVERSES["7"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF8" + INVERSES["F"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0" + INVERSES["9"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF6" + fi + if [ ${NR_CPUS} -ge 8 ]; then + INVERSES["11"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEE" + INVERSES["FF"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00" + INVERSES["F0"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F" + INVERSES["81"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF7E" + fi + if [ ${NR_CPUS} -ge 16 ]; then + INVERSES["1111"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFEEEE" + INVERSES["FFF"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFF000" + INVERSES["F0F"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F0" + INVERSES["F0F0"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFF0F0F" + INVERSES["0F0F"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F0" + INVERSES["FFFF"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFF0000" + INVERSES["FFFE"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFF0001" + INVERSES["8001"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFF7FFE" + fi + if [ ${NR_CPUS} -ge 32 ]; then + INVERSES["11111111"]="FFFFFFFFFFFFFFFFFFFFFFFFEEEEEEEE" + INVERSES["0F0F0F0F"]="FFFFFFFFFFFFFFFFFFFFFFFFF0F0F0F0" + INVERSES["F0F0F0F0"]="FFFFFFFFFFFFFFFFFFFFFFFF0F0F0F0F" + INVERSES["FFFFFFFF"]="FFFFFFFFFFFFFFFFFFFFFFFF00000000" + INVERSES["FFFFFFFE"]="FFFFFFFFFFFFFFFFFFFFFFFF00000001" + INVERSES["80000001"]="FFFFFFFFFFFFFFFFFFFFFFFF7FFFFFFE" + fi + if [ ${NR_CPUS} -ge 64 ]; then + INVERSES["1111111111111111"]="FFFFFFFFFFFFFFFFEEEEEEEEEEEEEEEE" + INVERSES["0F0F0F0F0F0F0F0F"]="FFFFFFFFFFFFFFFFF0F0F0F0F0F0F0F0" + INVERSES["F0F0F0F0F0F0F0F0"]="FFFFFFFFFFFFFFFF0F0F0F0F0F0F0F0F" + INVERSES["FFFFFFFFFFFFFFFF"]="FFFFFFFFFFFFFFFF0000000000000000" + INVERSES["FFFFFFFFFFFFFFFE"]="FFFFFFFFFFFFFFFF0000000000000001" + INVERSES["8000000000000001"]="FFFFFFFFFFFFFFFF7FFFFFFFFFFFFFFE" + fi + if [ ${NR_CPUS} -ge 128 ]; then + INVERSES["11111111111111111111111111111111"]="EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE" + INVERSES["0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F"]="F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0" + INVERSES["F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0"]="0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F" + INVERSES["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"]="00000000000000000000000000000000" + INVERSES["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"]="00000000000000000000000000000001" + INVERSES["80000000000000000000000000000001"]="7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE" + fi + + for CPUMAP in ${!INVERSES[@]}; do + EXPECTED=${INVERSES[${CPUMAP}]} + if [ ${NR_CPUS} -lt 128 ]; then + EXPECTED=$(echo ${EXPECTED} | cut --complement -c1-$((32-((${NR_CPUS}+3)/4)))) + fi + EXPECTED=$(echo ${EXPECTED} | sed -e "s/^0*//") + if [ -z ${EXPECTED} ]; then + EXPECTED="0" + fi + INVERSE=$(invert_cpumap ${CPUMAP} ${NR_CPUS}) + if [ "${INVERSE}" != "${EXPECTED}" ]; then + printf "\n" + echo "error: (invert_cpumap ${CPUMAP} ${NR_CPUS}) returned \"${INVERSE}\" instead of \"${EXPECTED}\"" + fi + printf "." + done + + printf "\n" +} + +for NR_CPUS in ${NR_CPUS_LIST[@]}; do + echo "NR_CPUS=${NR_CPUS}" + test_cpumap_to_cpulist ${NR_CPUS} + test_cpulist_to_cpumap ${NR_CPUS} + test_invert_cpumap ${NR_CPUS} + echo "" +done + +exit 0 diff --git a/utilities/worker-utils/worker-utils/ps-sched.sh b/utilities/worker-utils/worker-utils/ps-sched.sh new file mode 100755 index 00000000..0ce78b69 --- /dev/null +++ b/utilities/worker-utils/worker-utils/ps-sched.sh @@ -0,0 +1,26 @@ +#!/bin/bash +################################################################################ +# Copyright (c) 2013 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +################################################################################ +# +# ps-sched.sh -- gives detailed task listing with scheduling attributes +# -- this is cpu and scheduling intensive version (shell/taskset based) +# (note: does not print fields 'group' or 'timeslice') + +printf "%6s %6s %6s %1c %2s %4s %6s %4s %-24s %2s %-16s %s\n" "PID" "TID" "PPID" "S" "PO" "NICE" "RTPRIO" "PR" "AFFINITY" "P" "COMM" "COMMAND" +ps -eL -o pid=,lwp=,ppid=,state=,class=,nice=,rtprio=,priority=,psr=,comm=,command= | \ + while read pid tid ppid state policy nice rtprio priority psr comm command; do + bitmask=$(taskset -p $tid 2>/dev/null) + aff=${bitmask##*: } + if [ -z "${aff}" ]; then + aff="0x0" + else + aff="0x${aff}" + fi + printf "%6d %6d %6d %1c %2s %4s %6s %4d %-24s %2d %-16s %s\n" $pid $tid $ppid $state $policy $nice $rtprio $priority $aff $psr $comm "$command" + done + +exit 0 diff --git a/utilities/worker-utils/worker-utils/set-cpu-wakeup-latency.sh b/utilities/worker-utils/worker-utils/set-cpu-wakeup-latency.sh new file mode 100644 index 00000000..0efa13ec --- /dev/null +++ b/utilities/worker-utils/worker-utils/set-cpu-wakeup-latency.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# +# Copyright (c) 2017 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Purpose: set PM QoS resume latency constraints for CPUs. +# Usage: /usr/bin/set-cpu-wakeup-latency.sh policy cpulist +# policy may be either "low" or "high" to set appropriate latency. +# "low" means HALT (C1) is the deepest C-state we allow the CPU to enter. +# "high" means we allow the CPU to sleep as deeply as possible. +# cpulist is for specifying a numerical list of processors. +# It may contain multiple items, separated by comma, and ranges. +# For example, 0,5,7,9-11. + +# Define minimal path +PATH=/bin:/usr/bin:/usr/local/bin + +LOG_FUNCTIONS=${LOG_FUNCTIONS:-"/etc/init.d/log_functions.sh"} +CPUMAP_FUNCTIONS=${CPUMAP_FUNCTIONS:-"/etc/init.d/cpumap_functions.sh"} +[[ -e ${LOG_FUNCTIONS} ]] && source ${LOG_FUNCTIONS} +[[ -e ${CPUMAP_FUNCTIONS} ]] && source ${CPUMAP_FUNCTIONS} + +if [ $UID -ne 0 ]; then + log_error "$0 requires root or sudo privileges" + exit 1 +fi + +if [ "$#" -ne 2 ]; then + log_error "$0 requires policy and cpulist parameters" + exit 1 +fi + +POLICY=$1 +CPU_LIST=$2 +NUMBER_OF_CPUS=$(getconf _NPROCESSORS_CONF 2>/dev/null) +STATUS=1 + +for CPU_NUM in $(expand_sequence "$CPU_LIST" " "); do + # Check that we are not setting PM QoS policy for non-existing CPU + if [ "$CPU_NUM" -lt "0" ] || [ "$CPU_NUM" -ge "$NUMBER_OF_CPUS" ]; then + log_error "CPU number ${CPU_NUM} is invalid, available CPUs are 0-${NUMBER_OF_CPUS-1}" + exit 1 + fi + + # Obtain CPU wakeup latencies for all C-states available starting from operating state to deepest sleep + declare -a LIMITS=() + LIMITS+=($(cat /sys/devices/system/cpu/cpu${CPU_NUM}/cpuidle/state*/latency 2>/dev/null | xargs | sort)) + if [ ${#LIMITS[@]} -eq 0 ]; then + log_debug "Failed to get PM QoS latency limits for CPU ${CPU_NUM}" + fi + + # Select appropriate CPU wakeup latency based on "low" or "high" policy + case "${POLICY}" in + "low") + # Get first sleep state for "low" policy + if [ ${#LIMITS[@]} -eq 0 ]; then + LATENCY=1 + else + LATENCY=${LIMITS[1]} + fi + ;; + "high") + # Get deepest sleep state for "high" policy + if [ ${#LIMITS[@]} -eq 0 ]; then + LATENCY=1000 + else + LATENCY=${LIMITS[${#LIMITS[@]}-1]} + fi + ;; + *) + log_error "Policy is invalid, can be either low or high" + exit 1 + esac + + # Set the latency for paricular CPU + echo ${LATENCY} > /sys/devices/system/cpu/cpu${CPU_NUM}/power/pm_qos_resume_latency_us 2>/dev/null + RET_VAL=$? + if [ ${RET_VAL} -ne 0 ]; then + log_error "Failed to set PM QoS latency for CPU ${CPU_NUM}, rc=${RET_VAL}" + continue + else + log_debug "Succesfully set PM QoS latency for CPU ${CPU_NUM}, rc=${RET_VAL}" + STATUS=0 + fi +done + +exit ${STATUS} diff --git a/utilities/worker-utils/worker-utils/task_affinity_functions.sh b/utilities/worker-utils/worker-utils/task_affinity_functions.sh new file mode 100755 index 00000000..96f79121 --- /dev/null +++ b/utilities/worker-utils/worker-utils/task_affinity_functions.sh @@ -0,0 +1,322 @@ +#!/bin/bash +################################################################################ +# Copyright (c) 2017 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +################################################################################ +# Define minimal path +PATH=/bin:/usr/bin:/usr/local/bin + +. /etc/platform/platform.conf +LOG_FUNCTIONS=${LOG_FUNCTIONS:-"/etc/init.d/log_functions.sh"} +CPUMAP_FUNCTIONS=${CPUMAP_FUNCTIONS:-"/etc/init.d/cpumap_functions.sh"} +[[ -e ${LOG_FUNCTIONS} ]] && source ${LOG_FUNCTIONS} +[[ -e ${CPUMAP_FUNCTIONS} ]] && source ${CPUMAP_FUNCTIONS} + +# Enable debug logs and tag them +LOG_DEBUG=1 +TAG="TASKAFFINITY:" + +TASK_AFFINING_INCOMPLETE="/etc/platform/.task_affining_incomplete" +N_CPUS=$(cat /proc/cpuinfo 2>/dev/null | \ + awk '/^[pP]rocessor/ { n +=1 } END { print (n>0) ? n : 1}') +FULLSET_CPUS="0-"$((N_CPUS-1)) +FULLSET_MASK=$(cpulist_to_cpumap ${FULLSET_CPUS} ${N_CPUS}) +PLATFORM_CPUS=$(get_platform_cpu_list) +PLATFORM_CPULIST=$(get_platform_cpu_list| \ + perl -pe 's/(\d+)-(\d+)/join(",",$1..$2)/eg'| \ + sed 's/,/ /g') +VSWITCH_CPULIST=$(get_vswitch_cpu_list| \ + perl -pe 's/(\d+)-(\d+)/join(",",$1..$2)/eg'| \ + sed 's/,/ /g') +IDLE_MARK=95.0 +KERNEL=`uname -a` + +################################################################################ +# Check if a given core is one of the platform cores +################################################################################ +function is_platform_core { + local core=$1 + for CPU in ${PLATFORM_CPULIST}; do + if [ $core -eq $CPU ]; then + return 1 + fi + done + return 0 +} + +################################################################################ +# Check if a given core is one of the vswitch cores +################################################################################ +function is_vswitch_core { + local core=$1 + for CPU in ${VSWITCH_CPULIST}; do + if [ $core -eq $CPU ]; then + return 1 + fi + done + return 0 +} + +################################################################################ +# An audit and corrective action following a swact +################################################################################ +function audit_and_reaffine { + local mask=$1 + local cmd_str="" + local tasklist + + cmd_str="ps-sched.sh|awk '(\$9==\"$mask\") {print \$2}'" + + tasklist=($(eval $cmd_str)) + # log_debug "cmd str = $cmd_str" + log_debug "${TAG} There are ${#tasklist[@]} tasks to reaffine." + + for task in ${tasklist[@]}; do + taskset -acp ${PLATFORM_CPUS} $task &> /dev/null + rc=$? + [[ $rc -ne 0 ]] && log_error "Failed to set CPU affinity for pid $pid, rc=$rc" + done + tasklist=($(eval $cmd_str)) + [[ ${#tasklist[@]} -eq 0 ]] && return 0 || return 1 +} + +################################################################################ +# The following function is used to verify that any sleeping management tasks +# that are on non-platform cores can be migrated to platform cores as soon as +# they are scheduled. It can be invoked either manually or from goenableCompute +# script as a scheduled job (with a few minute delay) if desired. +# The induced tasks migration should be done after all VMs have been restored +# following a host reboot in AIO, hence the delay. +################################################################################ +function move_inactive_threads_to_platform_cores { + local tasklist + local cmd_str="" + + # Compile a list of non-kernel & non-vswitch/VM related threads that are not + # on platform cores. + # e.g. if the platform cpulist value is "0 8", the resulting command to be + # evaluated should look like this: + # ps-sched.sh|grep -v vswitch|awk '($10!=0 && $10!=8 && $3!=2) {if(NR>1)print $2}' + cmd_str="ps-sched.sh|grep -v vswitch|awk '(" + for cpu_num in ${PLATFORM_CPULIST}; do + cmd_str=$cmd_str"\$10!="${cpu_num}" && " + done + cmd_str=$cmd_str"\$3!=2) {if(NR>1)print \$2}'" + echo "selection string = $cmd_str" + tasklist=($(eval $cmd_str)) + log_debug "${TAG} There are ${#tasklist[@]} number of tasks to be moved." + + # These sleep tasks are stuck on the wrong core(s). They need to be woken up + # so they can be migrated to the right ones. Attaching and detaching strace + # momentarily to the task does the trick. + for task in ${tasklist[@]}; do + strace -p $task 2>/dev/null & + pid=$! + sleep 0.1 + kill -SIGINT $pid + done + tasklist=($(eval $cmd_str)) + [[ ${#tasklist[@]} -eq 0 ]] && return 0 || return 1 +} + +################################################################################ +# The following function is called by affine-platform.sh to affine tasks to +# all available cores during initial startup and subsequent host reboots. +################################################################################ +function affine_tasks_to_all_cores { + local pidlist + local rc=0 + + if [[ "${KERNEL}" == *" RT "* ]]; then + return 0 + fi + + log_debug "${TAG} Affining all tasks to CPU (${FULLSET_CPUS})" + + pidlist=$(ps --ppid 2 -p 2 --deselect -o pid= | awk '{ print $1; }') + for pid in ${pidlist[@]}; do + ppid=$(ps -o ppid= -p $pid |tr -d '[:space:]') + if [ -z $ppid ] || [ $ppid -eq 2 ]; then + continue + fi + log_debug "Affining pid $pid, parent pid = $ppid" + taskset --all-tasks --pid --cpu-list ${FULLSET_CPUS} $pid &> /dev/null + rc=$? + [[ $rc -ne 0 ]] && log_error "Failed to set CPU affinity for pid $pid, rc=$rc" + done + # Write the cpu list to a temp file which will be read and removed when + # the tasks are reaffined back to platform cores later on. + echo ${FULLSET_CPUS} > ${TASK_AFFINING_INCOMPLETE} + + return $rc +} + +################################################################################ +# The following function can be called by any platform service that needs to +# temporarily make use of idle VM cores to run a short-duration, service +# critical and cpu intensive operation in AIO. For instance, sm can levearage +# the idle cores to speed up swact activity. +# +# At the end of the operation, regarless of the result, the service must be +# calling function affine_tasks_to_platform_cores to re-affine platform tasks +# back to their assigned core(s). +# +# Kernel, vswitch and VM related tasks are untouched. +################################################################################ +function affine_tasks_to_idle_cores { + local cpulist + local cpuocc_list + local vswitch_pid + local pidlist + local idle_cpulist + local platform_cpus + local rc=0 + local cpu=0 + + if [ -f ${TASK_AFFINING_INCOMPLETE} ]; then + read cpulist < ${TASK_AFFINING_INCOMPLETE} + log_debug "${TAG} Tasks have already been affined to CPU ($cpulist)." + return 0 + fi + + if [[ "${KERNEL}" == *" RT "* ]]; then + return 0 + fi + + # Compile a list of cpus with idle percentage greater than 95% in the last + # 5 seconds. + cpuocc_list=($(sar -P ALL 1 5|grep Average|awk '{if(NR>2)print $8}')) + + for idle_value in ${cpuocc_list[@]}; do + is_vswitch_core $cpu + if [ $? -eq 1 ]; then + cpu=$(($cpu+1)) + continue + fi + + is_platform_core $cpu + if [ $? -eq 1 ]; then + # Platform core is added to the idle list by default + idle_cpulist=$idle_cpulist$cpu"," + else + # Non platform core is added to the idle list if it is more than 95% idle + [[ $(echo "$idle_value > ${IDLE_MARK}"|bc) -eq 1 ]] && idle_cpulist=$idle_cpulist$cpu"," + fi + cpu=$(($cpu+1)) + done + + idle_cpulist=$(echo $idle_cpulist|sed 's/.$//') + platform_affinity_mask=$(cpulist_to_cpumap ${PLATFORM_CPUS} ${N_CPUS} \ + |awk '{print tolower($0)}') + + log_debug "${TAG} Affining all tasks to idle CPU ($idle_cpulist)" + + vswitch_pid=$(pgrep vswitch) + pidlist=$(ps --ppid 2 -p 2 --deselect -o pid= | awk '{ print $1; }') + for pid in ${pidlist[@]}; do + ppid=$(ps -o ppid= -p $pid |tr -d '[:space:]') + if [ -z $ppid ] || [ $ppid -eq 2 ] || [ "$pid" = "$vswitch_pid" ]; then + continue + fi + pid_affinity_mask=$(taskset -p $pid | awk '{print $6}') + if [ "${pid_affinity_mask}" == "${platform_affinity_mask}" ]; then + # log_debug "Affining pid $pid to idle cores..." + taskset --all-tasks --pid --cpu-list $idle_cpulist $pid &> /dev/null + rc=$? + [[ $rc -ne 0 ]] && log_error "Failed to set CPU affinity for pid $pid, rc=$rc" + fi + done + + # Save the cpu list to the temp file which will be read and removed when + # tasks are reaffined to the platform cores later on. + echo $idle_cpulist > ${TASK_AFFINING_INCOMPLETE} + return $rc +} + +################################################################################ +# The following function is called by either: +# a) nova-compute wrapper script during AIO system initial bringup or reboot +# or +# b) sm at the end of swact sequence +# to re-affine management tasks back to the platform cores. +################################################################################ +function affine_tasks_to_platform_cores { + local cpulist + local pidlist + local rc=0 + local count=0 + + if [ ! -f ${TASK_AFFINING_INCOMPLETE} ]; then + dbg_str="${TAG} Either tasks have never been affined to all/idle cores or" + dbg_str=$dbg_str" they have already been reaffined to platform cores." + log_debug "$dbg_str" + return 0 + fi + + read cpulist < ${TASK_AFFINING_INCOMPLETE} + affinity_mask=$(cpulist_to_cpumap $cpulist ${N_CPUS}|awk '{print tolower($0)}') + + log_debug "${TAG} Reaffining tasks to platform cores (${PLATFORM_CPUS})..." + pidlist=$(ps --ppid 2 -p 2 --deselect -o pid= | awk '{ print $1; }') + for pid in ${pidlist[@]}; do + # log_debug "Processing pid $pid..." + pid_affinity_mask=$(taskset -p $pid | awk '{print $6}') + # Only management tasks need to be reaffined. Kernel, vswitch and VM related + # tasks were not affined previously so they should have different affinity + # mask(s). + if [ "${pid_affinity_mask}" == "${affinity_mask}" ]; then + count=$(($count+1)) + # log_debug "Affining pid $pid to platform cores..." + taskset --all-tasks --pid --cpu-list ${PLATFORM_CPUS} $pid &> /dev/null + rc=$? + [[ $rc -ne 0 ]] && log_error "Failed to set CPU affinity for pid $pid, rc=$rc" + fi + done + + # A workaround for lack of "end of swact" state + fullmask=$(echo ${FULLSET_MASK} | awk '{print tolower($0)}') + if [ "${affinity_mask}" != "${fullmask}" ]; then + log_debug "${TAG} Schedule an audit and cleanup" + (sleep 60; audit_and_reaffine "0x"$affinity_mask) & + fi + + rm -rf ${TASK_AFFINING_INCOMPLETE} + log_debug "${TAG} $count tasks were reaffined to platform cores." + + return $rc +} + +################################################################################ +# The following function can be leveraged by cron tasks +################################################################################ +function get_most_idle_core { + local cpuocc_list + local cpu=0 + local most_idle_value=${IDLE_MARK} + local most_idle_cpu=0 + + if [[ "${KERNEL}" == *" RT "* ]]; then + echo $cpu + return + fi + + cpuocc_list=($(sar -P ALL 1 5|grep Average|awk '{if(NR>2)print $8}')) + + for idle_value in ${cpuocc_list[@]}; do + is_vswitch_core $cpu + if [ $? -eq 1 ]; then + cpu=$(($cpu+1)) + continue + fi + + if [ $(echo "$idle_value > $most_idle_value"|bc) -eq 1 ]; then + most_idle_value=$idle_value + most_idle_cpu=$cpu + fi + cpu=$(($cpu+1)) + done + + echo $most_idle_cpu +} diff --git a/utilities/worker-utils/worker-utils/topology b/utilities/worker-utils/worker-utils/topology new file mode 100644 index 00000000..9f6e26fc --- /dev/null +++ b/utilities/worker-utils/worker-utils/topology @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +python /usr/bin/topology.pyc diff --git a/utilities/worker-utils/worker-utils/topology.py b/utilities/worker-utils/worker-utils/topology.py new file mode 100755 index 00000000..08ed4c52 --- /dev/null +++ b/utilities/worker-utils/worker-utils/topology.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python +################################################################################ +# Copyright (c) 2013 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +################################################################################ +# +# topology.py -- gives a summary of logical cpu enumeration, +# sockets, cores per package, threads per core, +# total memory, and numa nodes + +from __future__ import print_function +import os +import sys +import re + +class Topology(object): + """ Build up topology information. + (i.e. logical cpu topology, NUMA nodes, memory) + """ + + def __init__(self): + self.num_cpus = 0 + self.num_nodes = 0 + self.num_sockets = 0 + self.num_cores_per_pkg = 0 + self.num_threads_per_core = 0 + + self.topology = {} + self.topology_idx = {} + self.total_memory_MiB = 0 + self.total_memory_nodes_MiB = [] + + self._get_cpu_topology() + self._get_total_memory_MiB() + self._get_total_memory_nodes_MiB() + + def _get_cpu_topology(self): + '''Enumerate logical cpu topology based on parsing /proc/cpuinfo + as function of socket_id, core_id, and thread_id. This updates + topology and reverse index topology_idx mapping. + + :param self + :updates self.num_cpus - number of logical cpus + :updates self.num_nodes - number of sockets; maps to number of numa nodes + :updates self.topology[socket_id][core_id][thread_id] = cpu + :updates self.topology_idx[cpu] = {'s': socket_id, 'c': core_id, 't': thread_id} + :returns None + ''' + + self.num_cpus = 0 + self.num_nodes = 0 + self.num_sockets = 0 + self.num_cores = 0 + self.num_threads = 0 + self.topology = {} + self.topology_idx = {} + + Thread_cnt = {} + cpu = socket_id = core_id = thread_id = -1 + re_processor = re.compile(r'^[Pp]rocessor\s+:\s+(\d+)') + re_socket = re.compile(r'^physical id\s+:\s+(\d+)') + re_core = re.compile(r'^core id\s+:\s+(\d+)') + + with open('/proc/cpuinfo', 'r') as infile: + for line in infile: + + match = re_processor.search(line) + if match: + cpu = int(match.group(1)) + socket_id = -1; core_id = -1; thread_id = -1 + self.num_cpus += 1 + continue + + match = re_socket.search(line) + if match: + socket_id = int(match.group(1)) + continue + + match = re_core.search(line) + if match: + core_id = int(match.group(1)) + + if socket_id not in Thread_cnt: + Thread_cnt[socket_id] = {} + if core_id not in Thread_cnt[socket_id]: + Thread_cnt[socket_id][core_id] = 0 + else: + Thread_cnt[socket_id][core_id] += 1 + thread_id = Thread_cnt[socket_id][core_id] + + if socket_id not in self.topology: + self.topology[socket_id] = {} + if core_id not in self.topology[socket_id]: + self.topology[socket_id][core_id] = {} + + self.topology[socket_id][core_id][thread_id] = cpu + self.topology_idx[cpu] = {'s': socket_id, 'c': core_id, 't': thread_id} + continue + self.num_nodes = len(self.topology.keys()) + + # In the case topology not detected, hard-code structures + if self.num_nodes == 0: + n_sockets, n_cores, n_threads = (1, self.num_cpus, 1) + self.topology = {} + for socket_id in range(n_sockets): + self.topology[socket_id] = {} + for core_id in range(n_cores): + self.topology[socket_id][core_id] = {} + for thread_id in range(n_threads): + self.topology[socket_id][core_id][thread_id] = 0 + # Define Thread-Socket-Core order for logical cpu enumeration + self.topology_idx = {} + cpu = 0 + for thread_id in range(n_threads): + for socket_id in range(n_sockets): + for core_id in range(n_cores): + self.topology[socket_id][core_id][thread_id] = cpu + self.topology_idx[cpu] = {'s': socket_id, 'c': core_id, 't': thread_id} + cpu += 1 + self.num_nodes = len(self.topology.keys()) + + self.num_sockets = len(self.topology.keys()) + self.num_cores_per_pkg = len(self.topology[0].keys()) + self.num_threads_per_core = len(self.topology[0][0].keys()) + + return None + + def _get_total_memory_MiB(self): + """Get the total memory for VMs (MiB). + + :updates: total memory for VMs (MiB) + + """ + + self.total_memory_MiB = 0 + + # Total memory + try: + m = open('/proc/meminfo').read().split() + idx_Total = m.index('MemTotal:') + 1 + self.total_memory_MiB = int(m[idx_Total]) / 1024 + except IOError: + # silently ignore IO errors (eg. file missing) + pass + return None + + def _get_total_memory_nodes_MiB(self): + """Get the total memory per numa node for VMs (MiB). + + :updates: total memory per numa node for VMs (MiB) + + """ + + self.total_memory_nodes_MiB = [] + + # Memory of each numa node (MiB) + for node in range(self.num_nodes): + Total_MiB = 0 + + meminfo = "/sys/devices/system/node/node%d/meminfo" % node + try: + m = open(meminfo).read().split() + idx_Total = m.index('MemTotal:') + 1 + Total_MiB = int(m[idx_Total]) / 1024 + except IOError: + # silently ignore IO errors (eg. file missing) + pass + + self.total_memory_nodes_MiB.append(Total_MiB) + return None + + def _print_cpu_topology(self): + '''Print logical cpu topology enumeration as function of: + socket_id, core_id, and thread_id. + + :param self + :returns None + ''' + + cpu_list = self.topology_idx.keys() + cpu_list.sort() + total_memory_GiB = self.total_memory_MiB/1024.0 + + print('TOPOLOGY:') + print('%16s : %5d' % ('logical cpus', self.num_cpus)) + print('%16s : %5d' % ('sockets', self.num_sockets)) + print('%16s : %5d' % ('cores_per_pkg', self.num_cores_per_pkg)) + print('%16s : %5d' % ('threads_per_core', self.num_threads_per_core)) + print('%16s : %5d' % ('numa_nodes', self.num_nodes)) + print('%16s : %5.2f %s' % ('total_memory', total_memory_GiB, 'GiB')) + print('%16s :' % ('memory_per_node'), end=' ') + for node in range(self.num_nodes): + node_memory_GiB = self.total_memory_nodes_MiB[node]/1024.0 + print('%5.2f' % (node_memory_GiB), end=' ') + print('%s' % ('GiB')) + print('') + + print('LOGICAL CPU TOPOLOGY:') + print("%9s :" % 'cpu_id', end=' ') + for cpu in cpu_list: + print("%3d" % cpu, end=' ') + print('') + print("%9s :" % 'socket_id', end=' ') + for cpu in cpu_list: + socket_id = self.topology_idx[cpu]['s'] + print("%3d" % socket_id, end=' ') + print('') + print("%9s :" % 'core_id', end=' ') + for cpu in cpu_list: + core_id = self.topology_idx[cpu]['c'] + print("%3d" % core_id, end=' ') + print('') + print("%9s :" % 'thread_id', end=' ') + for cpu in cpu_list: + thread_id = self.topology_idx[cpu]['t'] + print("%3d" % thread_id, end=' ') + print('') + print('') + + print('CORE TOPOLOGY:') + print("%6s %9s %7s %9s %s" % ('cpu_id', 'socket_id', 'core_id', 'thread_id', 'affinity')) + for cpu in cpu_list: + affinity = 1<::. +# +# Example: To reserve 1500MB and 1 core on NUMA node0, and 1500MB and 1 core +# on NUMA node1, the variable must be specified as follows. +# WORKER_BASE_MEMORY=("node0:1500MB:1" "node1:1500MB:1") +# +################################################################################ +WORKER_BASE_RESERVED=("node0:8000MB:1" "node1:2000MB:0" "node2:2000MB:0" "node3:2000MB:0") + +################################################################################ +# +# List of HugeTLB memory descriptors to configure. Each array element +# consists of a 3-tuple descriptor formatted as: ::. +# The NUMA node specified must exist and the HugeTLB pagesize must be a valid +# value such as 2048kB or 1048576kB. +# +# For example, to request 256 x 2MB HugeTLB pages on NUMA node0 and node1 the +# variable must be specified as follows. +# WORKER_VSWITCH_MEMORY=("node0:2048kB:256" "node1:2048kB:256") +# +################################################################################ +WORKER_VSWITCH_MEMORY=("node0:1048576kB:1" "node1:1048576kB:1" "node2:1048576kB:1" "node3:1048576kB:1") + +################################################################################ +# +# List of VSWITCH physical cores reserved for VSWITCH applications. +# +# Example: To reserve 2 cores on NUMA node0, and 2 cores on NUMA node1, the +# variable must be specified as follows. +# WORKER_VSWITCH_CORES=("node0:2" "node1:2") +# +################################################################################ +WORKER_VSWITCH_CORES=("node0:2" "node1:0" "node2:0" "node3:0")