starlingx/virtual-deployment
Code Issues Proposed changes

Libvirt: CPU and Memory value configurable

Browse Source 
Add the ability for the user to change how many cores and memory are
allocated per node. Each node may have any number of cores or memory
set, with which their values are used by sourcing the file:
'source readconfig.sh <yaml file>'

Test plan:
PASS: regression tests passed
PASS: sanity tests passed
PASS: no tox, flake8 or pylint errors
PASS: value succesfully set from config file
PASS: defaults used when no config file is sourced

Story: 2010816
Task: 48398
Task: 48586

Change-Id: Ia2f7df44c872fac41ac6376ef3fb00062624ac22
Signed-off-by: Bailey Henry <Henry.Bailey@windriver.com>
This commit is contained in:
Bailey Henry 2023-08-10 16:42:25 -04:00 committed by Michel Thebeau
parent 8c64195f67
commit 2066a90fa5
9 changed files with 144 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
libvirt/config.py
View File

@ -18,7 +18,7 @@ import yaml
BRIDGE_LIMIT = 9
# Fields that should be set in a Yaml configuration file
SUPPORTED_HOST_KEYS = ['disk']
SUPPORTED_HOST_KEYS = ['disk', 'cpu', 'mem']
GEN_FIELDS = ('bridge_interface', 'controller', 'worker',
'domain_dir', 'storage', 'default_disk')
@ -27,9 +27,12 @@ IP_FIELDS = ('ext_network', 'ext_IP')
HOST_FIELDS = ('controllerlist', 'workerlist', 'storagelist')
NUMBER_FIELDS = ('aio_default_cpu', 'aio_default_mem', 'default_cpu',
'default_mem')
NODES_NUM = ('worker_nodes_num', 'storage_nodes_num')
FIELDS = GEN_FIELDS + IP_FIELDS + HOST_FIELDS + NODES_NUM
FIELDS = NODES_NUM + GEN_FIELDS + IP_FIELDS + HOST_FIELDS + NUMBER_FIELDS
HELP_TEXT = """
use:
@ -39,6 +42,14 @@ use:
"""
def number_validate(value, field):
error = 0
if not isinstance(value, int):
print('%s not valid for %s' % (value, field), file=sys.stderr)
error += 1
return error
# Allows the user to see how this script is used
def print_help():
print(HELP_TEXT, file=sys.stderr)
@ -72,6 +83,8 @@ def host_key_check(listdata, printname):
# Iterated through lists and returns specific fields
def item_from_list(data, fields, num):
item = None
try:
listdata = data[fields[0]]
key2 = fields[2]
@ -82,8 +95,8 @@ def item_from_list(data, fields, num):
except (IndexError, KeyError):
# Node index is not config yaml
# Specified data key is not in config yaml
item = data['default_disk']
if key2 == 'disk':
item = data['default_disk']
else:
print('ValueError: key %s is not supported' % key2,
file=sys.stderr)
@ -92,7 +105,6 @@ def item_from_list(data, fields, num):
except TypeError as e:
print('Sanity: incorrect key type for list or dictionary read:'
' % s ' % e, file=sys.stderr)
item = data['default_disk']
return(item)
@ -108,6 +120,27 @@ def check_path(value, field):
return 0
# Validation for checking cpu and memory values
def host_key_validate(data, field, value):
errnum = 0
for i in range(len(value)):
for key in SUPPORTED_HOST_KEYS:
fieldlist = [field, i, key]
item = item_from_list(data, fieldlist, i)
if key == 'disk':
err = check_path(item, field)
if not err == 0:
print('%s index %s has bad path' % (field, i),
file=sys.stderr)
errnum += 1
elif key in ('cpu', 'mem'):
if not isinstance(item, int):
print('%s is not valid for %s' %
(item, field), file=sys.stderr)
errnum += 1
return errnum
# Validation for checking general fields
def general_validate(value, field):
if (field == 'bridge_interface' and len(value) > BRIDGE_LIMIT):
@ -146,21 +179,12 @@ def host_validate(value, field, data):
errnum = 0
if isinstance(value, list):
# This is a host list, check if the keys are recognized
errnum = host_key_check(value, field)
errnum += host_key_check(value, field)
if errnum:
return errnum
# validate each recognized key
for i in range(len(value)):
for key in SUPPORTED_HOST_KEYS:
fieldlist = [field, i, key]
item = item_from_list(data, fieldlist, i)
if key == 'disk':
err = check_path(item, field)
if not err == 0:
print('%s index %s has bad path' % (field, i),
file=sys.stderr)
errnum += err
errnum += host_key_validate(data, field, value)
else:
print(field + ' is not a list',
file=sys.stderr)
@ -233,10 +257,13 @@ def validator(config_file):
try:
value = data[field]
except KeyError:
print(' %s does not exist' % field, file=sys.stderr)
print('%s does not exist' % field, file=sys.stderr)
ERROR += 1
continue
if (field in NUMBER_FIELDS):
ERROR += number_validate(value, field)
# Checking that most fields are alphanumerical
if (field in GEN_FIELDS):
ERROR += general_validate(value, field)
@ -278,6 +305,10 @@ def readvalue(config_file, fieldlist):
sys.exit(1)
result = item_from_list(data, fieldlist, num)
if not result:
sys.exit(1)
print(result)
else:
print('TypeError: %s is not list' % fieldlist[0], file=sys.stderr)

8
libvirt/controller.xml
View File

@ -1,8 +1,8 @@
<domain type='kvm' id='164'>
<name>NAME</name>
<memory unit='KiB'>16777216</memory>
<currentMemory unit='KiB'>16777216</currentMemory>
<vcpu placement='static'>4</vcpu>
<memory unit='GiB'>%MEM%</memory>
<currentMemory unit='GiB'>%MEM%</currentMemory>
<vcpu placement='static'>%CPU%</vcpu>
<resource>
<partition>/machine</partition>
</resource>
@ -16,7 +16,7 @@
</features>
<cpu match='exact'>
<model fallback='forbid'>Nehalem</model>
<topology sockets='1' cores='4' threads='1'/>
<topology sockets='1' cores='%CPU%' threads='1'/>
<feature policy='optional' name='vmx'/>
<feature policy='optional' name='svm'/>
</cpu>

8
libvirt/controller_allinone.xml
View File

@ -1,8 +1,8 @@
<domain type='kvm' id='164'>
<name>NAME</name>
<memory unit='GiB'>18</memory>
<currentMemory unit='GiB'>18</currentMemory>
<vcpu placement='static'>6</vcpu>
<memory unit='GiB'>%MEM%</memory>
<currentMemory unit='GiB'>%MEM%</currentMemory>
<vcpu placement='static'>%CPU%</vcpu>
<resource>
<partition>/machine</partition>
</resource>
@ -16,7 +16,7 @@
</features>
<cpu match='exact'>
<model fallback='forbid'>Nehalem</model>
<topology sockets='1' cores='6' threads='1'/>
<topology sockets='1' cores='%CPU%' threads='1'/>
<feature policy='optional' name='vmx'/>
<feature policy='optional' name='svm'/>
</cpu>

16
libvirt/default.yaml
View File

@ -9,12 +9,28 @@ worker_nodes_num: 1
storage_nodes_num: 1
domain_dir: 'vms'
default_disk: /var/lib/libvirt/images
aio_default_cpu: 6
aio_default_mem: 18
default_cpu: 4
default_mem: 16
controllerlist:
- disk: /var/lib/libvirt/images
cpu: 6
mem: 18
- disk: /var/lib/libvirt/images
cpu: 6
mem: 18
workerlist:
- disk: /var/lib/libvirt/images
cpu: 4
mem: 16
- disk: /var/lib/libvirt/images
cpu: 4
mem: 16
storagelist:
- disk: /var/lib/libvirt/images
cpu: 4
mem: 16
- disk: /var/lib/libvirt/images
cpu: 4
mem: 16

54
libvirt/functions.sh
View File

@ -59,6 +59,48 @@ get_disk(){
echo $diskdir
}
get_cpu(){
local field=$1
local num=$2
local cpu_count
if [ -n "$CONFIG_FILE" ] && [ -f "$CONFIG_FILE" ]; then
cpu_count=$( ./config.py $CONFIG_FILE $field $num cpu )
fi
if [ -z "$cpu_count" ]; then
if [ $field == 'controllerlist' ] && ([ $CONFIGURATION == \
'simplex' ] || [ $CONFIGURATION == 'duplex' ]); then
cpu_count=$AIO_DEFAULT_CPU
else
cpu_count=$DEFAULT_CPU
fi
fi
echo $cpu_count
}
get_mem(){
local field=$1
local num=$2
local memory
if [ -n "$CONFIG_FILE" ] && [ -f "$CONFIG_FILE" ]; then
memory=$( ./config.py $CONFIG_FILE $field $num mem )
fi
if [ -z "$memory" ]; then
if [ $field == 'controllerlist' ] && ([ $CONFIGURATION == \
'simplex' ] || [ $CONFIGURATION == 'duplex' ]); then
memory=$AIO_DEFAULT_MEM
else
memory=$DEFAULT_MEM
fi
fi
echo $memory
}
# delete a node's disk file in a safe way
delete_disk() {
local fpath="$1"
@ -113,6 +155,8 @@ create_controller() {
CONTROLLER_NODE=${CONFIGURATION}-${CONTROLLER}-${i}
DOMAIN_FILE=${DOMAIN_DIRECTORY}/${CONTROLLER_NODE}.xml
CPU="$( get_cpu controllerlist $i )"
MEM="$( get_mem controllerlist $i )"
DISK_LOCATION="$( get_disk controllerlist $i )"
mkdir -p "${DISK_LOCATION}"
if [ $? -ne 0 ]; then
@ -130,6 +174,8 @@ create_controller() {
fi
sed -i -e "
s,NAME,${CONTROLLER_NODE},
s,%CPU%,${CPU},
s,%MEM%,${MEM},
s,DISK0,${DISK_LOCATION}/${CONTROLLER_NODE}-0.img,
s,DISK1,${DISK_LOCATION}/${CONTROLLER_NODE}-1.img,
s,%BR1%,${BRIDGE_INTERFACE}1,
@ -199,9 +245,13 @@ create_node() {
if [ $IDENTITY == 'worker' ]; then
NODE="${CONFIGURATION}-${WORKER}-${INDEX}"
CPU="$( get_cpu workerlist $INDEX )"
MEM="$( get_mem workerlist $INDEX )"
DISK_LOCATION="$( get_disk workerlist $INDEX )"
elif [ $IDENTITY == 'storage' ]; then
NODE="${CONFIGURATION}-${STORAGE}-${INDEX}"
CPU="$( get_cpu storagelist $INDEX )"
MEM="$( get_mem storagelist $INDEX )"
DISK_LOCATION="$( get_disk storagelist $INDEX )"
fi
mkdir -p "${DISK_LOCATION}"
@ -217,7 +267,9 @@ create_node() {
sudo qemu-img create -f qcow2 ${DISK_LOCATION}/${NODE}-1.img 200G
cp ${IDENTITY}.xml ${DOMAIN_FILE}
sed -i -e "
s,NAME,${NODE},;
s,NAME,${NODE},
s,%CPU%,${CPU},
s,%MEM%,${MEM},
s,DISK0,${DISK_LOCATION}/${NODE}-0.img,;
s,DISK1,${DISK_LOCATION}/${NODE}-1.img,
s,%BR1%,${BRIDGE_INTERFACE}1,

5
libvirt/readconfig.sh
View File

@ -20,3 +20,8 @@ export WORKER_NODES_NUMBER="$( ${GET_CFG} worker_nodes_num )"
export STORAGE_NODES_NUMBER="$( ${GET_CFG} storage_nodes_num )"
export DOMAIN_DIRECTORY="$( ${GET_CFG} domain_dir )"
export DEFAULT_DISK_DIR="$( ${GET_CFG} default_disk )"
export DEFAULT_CPU="$( ${GET_CFG} default_cpu )"
export DEFAULT_MEM="$( ${GET_CFG} default_mem )"
export AIO_DEFAULT_CPU="$( ${GET_CFG} aio_default_cpu )"
export AIO_DEFAULT_MEM="$( ${GET_CFG} aio_default_mem )"

6
libvirt/set_defaults.sh
View File

@ -21,5 +21,11 @@ export STORAGE=${STORAGE:-storage}
export WORKER_NODES_NUMBER=${WORKER_NODES_NUMBER:-1}
export STORAGE_NODES_NUMBER=${STORAGE_NODES_NUMBER:-1}
export DEFAULT_CPU=${DEFAULT_CPU:-4}
export DEFAULT_MEM=${DEFAULT_MEM:-16}
export AIO_DEFAULT_CPU=${AIO_DEFAULT_CPU:-6}
export AIO_DEFAULT_MEM=${AIO_DEFAULT_MEM:-18}
export DOMAIN_DIRECTORY=${DOMAIN_DIRECTORY:-vms}
export DEFAULT_DISK_DIR=${DEFAULT_DISK_DIR:-/var/lib/libvirt/images}

8
libvirt/storage.xml
View File

@ -1,8 +1,8 @@
<domain type='kvm' id='187'>
<name>NAME</name>
<memory unit='KiB'>16777216</memory>
<currentMemory unit='KiB'>16777216</currentMemory>
<vcpu placement='static'>4</vcpu>
<memory unit='GiB'>%MEM%</memory>
<currentMemory unit='GiB'>%MEM%</currentMemory>
<vcpu placement='static'>%CPU%</vcpu>
<resource>
<partition>/machine</partition>
</resource>
@ -16,7 +16,7 @@
</features>
<cpu match='exact'>
<model fallback='forbid'>Nehalem</model>
<topology sockets='1' cores='4' threads='1'/>
<topology sockets='1' cores='%CPU%' threads='1'/>
<feature policy='require' name='vmx'/>
<feature policy='optional' name='svm'/>
</cpu>

8
libvirt/worker.xml
View File

@ -1,8 +1,8 @@
<domain type='kvm' id='187'>
<name>NAME</name>
<memory unit='KiB'>16777216</memory>
<currentMemory unit='KiB'>16777216</currentMemory>
<vcpu placement='static'>4</vcpu>
<memory unit='GiB'>%MEM%</memory>
<currentMemory unit='GiB'>%MEM%</currentMemory>
<vcpu placement='static'>%CPU%</vcpu>
<resource>
<partition>/machine</partition>
</resource>
@ -16,7 +16,7 @@
</features>
<cpu match='exact'>
<model fallback='forbid'>Nehalem</model>
<topology sockets='1' cores='4' threads='1'/>
<topology sockets='1' cores='%CPU%' threads='1'/>
<feature policy='require' name='vmx'/>
<feature policy='optional' name='svm'/>
</cpu>