Libvirt: CPU and Memory value configurable

Add the ability for the user to change how many cores and memory are
allocated per node. Each node may have any number of cores or memory
set, with which their values are used by sourcing the file:
'source readconfig.sh <yaml file>'

Test plan:
PASS: regression tests passed
PASS: sanity tests passed
PASS: no tox, flake8 or pylint errors
PASS: value succesfully set from config file
PASS: defaults used when no config file is sourced

Story: 2010816
Task: 48398
Task: 48586

Change-Id: Ia2f7df44c872fac41ac6376ef3fb00062624ac22
Signed-off-by: Bailey Henry <Henry.Bailey@windriver.com>

libvirt/config.py

@@ -18,7 +18,7 @@ import yaml
 BRIDGE_LIMIT = 9
 
 # Fields that should be set in a Yaml configuration file
-SUPPORTED_HOST_KEYS = ['disk']
+SUPPORTED_HOST_KEYS = ['disk', 'cpu', 'mem']
 
 GEN_FIELDS = ('bridge_interface', 'controller', 'worker',
               'domain_dir', 'storage', 'default_disk')
@@ -27,9 +27,12 @@ IP_FIELDS = ('ext_network', 'ext_IP')
 
 HOST_FIELDS = ('controllerlist', 'workerlist', 'storagelist')
 
+NUMBER_FIELDS = ('aio_default_cpu', 'aio_default_mem', 'default_cpu',
+                 'default_mem')
+
 NODES_NUM = ('worker_nodes_num', 'storage_nodes_num')
 
-FIELDS = GEN_FIELDS + IP_FIELDS + HOST_FIELDS + NODES_NUM
+FIELDS = NODES_NUM + GEN_FIELDS + IP_FIELDS + HOST_FIELDS + NUMBER_FIELDS
 
 HELP_TEXT = """
 use:
@@ -39,6 +42,14 @@ use:
 """
 
 
+def number_validate(value, field):
+    error = 0
+    if not isinstance(value, int):
+        print('%s not valid for %s' % (value, field), file=sys.stderr)
+        error += 1
+    return error
+
+
 # Allows the user to see how this script is used
 def print_help():
     print(HELP_TEXT, file=sys.stderr)
@@ -72,6 +83,8 @@ def host_key_check(listdata, printname):
 
 # Iterated through lists and returns specific fields
 def item_from_list(data, fields, num):
+    item = None
+
     try:
         listdata = data[fields[0]]
         key2 = fields[2]
@@ -82,8 +95,8 @@ def item_from_list(data, fields, num):
             except (IndexError, KeyError):
                 # Node index is not config yaml
                 # Specified data key is not in config yaml
+                if key2 == 'disk':
                     item = data['default_disk']
-
         else:
             print('ValueError: key %s is not supported' % key2,
                   file=sys.stderr)
@@ -92,7 +105,6 @@ def item_from_list(data, fields, num):
     except TypeError as e:
         print('Sanity: incorrect key type for list or dictionary read:'
               ' % s ' % e, file=sys.stderr)
-        item = data['default_disk']
     return(item)
 
 
@@ -108,6 +120,27 @@ def check_path(value, field):
     return 0
 
 
+# Validation for checking cpu and memory values
+def host_key_validate(data, field, value):
+    errnum = 0
+    for i in range(len(value)):
+        for key in SUPPORTED_HOST_KEYS:
+            fieldlist = [field, i, key]
+            item = item_from_list(data, fieldlist, i)
+            if key == 'disk':
+                err = check_path(item, field)
+                if not err == 0:
+                    print('%s index %s has bad path' % (field, i),
+                          file=sys.stderr)
+                    errnum += 1
+            elif key in ('cpu', 'mem'):
+                if not isinstance(item, int):
+                    print('%s is not valid for %s' %
+                          (item, field), file=sys.stderr)
+                    errnum += 1
+    return errnum
+
+
 # Validation for checking general fields
 def general_validate(value, field):
     if (field == 'bridge_interface' and len(value) > BRIDGE_LIMIT):
@@ -146,21 +179,12 @@ def host_validate(value, field, data):
     errnum = 0
     if isinstance(value, list):
         # This is a host list, check if the keys are recognized
-        errnum = host_key_check(value, field)
+        errnum += host_key_check(value, field)
         if errnum:
             return errnum
 
         # validate each recognized key
-        for i in range(len(value)):
+        errnum += host_key_validate(data, field, value)
-            for key in SUPPORTED_HOST_KEYS:
-                fieldlist = [field, i, key]
-                item = item_from_list(data, fieldlist, i)
-                if key == 'disk':
-                    err = check_path(item, field)
-                    if not err == 0:
-                        print('%s index %s has bad path' % (field, i),
-                              file=sys.stderr)
-                        errnum += err
     else:
         print(field + ' is not a list',
               file=sys.stderr)
@@ -233,10 +257,13 @@ def validator(config_file):
         try:
             value = data[field]
         except KeyError:
-            print(' %s does not exist' % field, file=sys.stderr)
+            print('%s does not exist' % field, file=sys.stderr)
             ERROR += 1
             continue
 
+        if (field in NUMBER_FIELDS):
+            ERROR += number_validate(value, field)
+
         # Checking that most fields are alphanumerical
         if (field in GEN_FIELDS):
             ERROR += general_validate(value, field)
@@ -278,6 +305,10 @@ def readvalue(config_file, fieldlist):
                 sys.exit(1)
 
             result = item_from_list(data, fieldlist, num)
+
+            if not result:
+                sys.exit(1)
+
             print(result)
         else:
             print('TypeError: %s is not list' % fieldlist[0], file=sys.stderr)

libvirt/controller.xml

@@ -1,8 +1,8 @@
 <domain type='kvm' id='164'>
   <name>NAME</name>
-  <memory unit='KiB'>16777216</memory>
+  <memory unit='GiB'>%MEM%</memory>
-  <currentMemory unit='KiB'>16777216</currentMemory>
+  <currentMemory unit='GiB'>%MEM%</currentMemory>
-  <vcpu placement='static'>4</vcpu>
+  <vcpu placement='static'>%CPU%</vcpu>
   <resource>
     <partition>/machine</partition>
   </resource>
@@ -16,7 +16,7 @@
   </features>
   <cpu match='exact'>
     <model fallback='forbid'>Nehalem</model>
-    <topology sockets='1' cores='4' threads='1'/>
+    <topology sockets='1' cores='%CPU%' threads='1'/>
     <feature policy='optional' name='vmx'/>
     <feature policy='optional' name='svm'/>
   </cpu>

libvirt/controller_allinone.xml

@@ -1,8 +1,8 @@
 <domain type='kvm' id='164'>
   <name>NAME</name>
-  <memory unit='GiB'>18</memory>
+  <memory unit='GiB'>%MEM%</memory>
-  <currentMemory unit='GiB'>18</currentMemory>
+  <currentMemory unit='GiB'>%MEM%</currentMemory>
-  <vcpu placement='static'>6</vcpu>
+  <vcpu placement='static'>%CPU%</vcpu>
   <resource>
     <partition>/machine</partition>
   </resource>
@@ -16,7 +16,7 @@
   </features>
   <cpu match='exact'>
     <model fallback='forbid'>Nehalem</model>
-    <topology sockets='1' cores='6' threads='1'/>
+    <topology sockets='1' cores='%CPU%' threads='1'/>
     <feature policy='optional' name='vmx'/>
     <feature policy='optional' name='svm'/>
   </cpu>

libvirt/default.yaml

@@ -9,12 +9,28 @@ worker_nodes_num: 1
 storage_nodes_num: 1
 domain_dir: 'vms'
 default_disk: /var/lib/libvirt/images
+aio_default_cpu: 6
+aio_default_mem: 18
+default_cpu: 4
+default_mem: 16
 controllerlist:
   - disk: /var/lib/libvirt/images
+    cpu: 6
+    mem: 18
   - disk: /var/lib/libvirt/images
+    cpu: 6
+    mem: 18
 workerlist:
   - disk: /var/lib/libvirt/images
+    cpu: 4
+    mem: 16
   - disk: /var/lib/libvirt/images
+    cpu: 4
+    mem: 16
 storagelist:
   - disk: /var/lib/libvirt/images
+    cpu: 4
+    mem: 16
   - disk: /var/lib/libvirt/images
+    cpu: 4
+    mem: 16

libvirt/functions.sh

@@ -59,6 +59,48 @@ get_disk(){
     echo $diskdir
 }
 
+get_cpu(){
+
+    local field=$1
+    local num=$2
+    local cpu_count
+
+    if [ -n "$CONFIG_FILE" ] && [ -f "$CONFIG_FILE" ]; then
+        cpu_count=$( ./config.py $CONFIG_FILE $field $num cpu )
+    fi
+
+    if [ -z "$cpu_count" ]; then
+        if [ $field == 'controllerlist' ] && ([ $CONFIGURATION == \
+        'simplex' ] || [ $CONFIGURATION == 'duplex' ]); then
+            cpu_count=$AIO_DEFAULT_CPU
+        else
+            cpu_count=$DEFAULT_CPU
+        fi
+    fi
+    echo $cpu_count
+}
+
+get_mem(){
+
+    local field=$1
+    local num=$2
+    local memory
+
+    if [ -n "$CONFIG_FILE" ] && [ -f "$CONFIG_FILE" ]; then
+        memory=$( ./config.py $CONFIG_FILE $field $num mem )
+    fi
+
+    if [ -z "$memory" ]; then
+        if [ $field == 'controllerlist' ] && ([ $CONFIGURATION == \
+        'simplex' ] || [ $CONFIGURATION == 'duplex' ]); then
+            memory=$AIO_DEFAULT_MEM
+        else
+            memory=$DEFAULT_MEM
+        fi
+    fi
+    echo $memory
+}
+
 # delete a node's disk file in a safe way
 delete_disk() {
     local fpath="$1"
@@ -113,6 +155,8 @@ create_controller() {
         CONTROLLER_NODE=${CONFIGURATION}-${CONTROLLER}-${i}
         DOMAIN_FILE=${DOMAIN_DIRECTORY}/${CONTROLLER_NODE}.xml
 
+        CPU="$( get_cpu controllerlist $i )"
+        MEM="$( get_mem controllerlist $i )"
         DISK_LOCATION="$( get_disk controllerlist $i )"
         mkdir -p "${DISK_LOCATION}"
         if [ $? -ne 0 ]; then
@@ -130,6 +174,8 @@ create_controller() {
         fi
         sed -i -e "
             s,NAME,${CONTROLLER_NODE},
+            s,%CPU%,${CPU},
+            s,%MEM%,${MEM},
             s,DISK0,${DISK_LOCATION}/${CONTROLLER_NODE}-0.img,
             s,DISK1,${DISK_LOCATION}/${CONTROLLER_NODE}-1.img,
             s,%BR1%,${BRIDGE_INTERFACE}1,
@@ -199,9 +245,13 @@ create_node() {
 
     if [ $IDENTITY == 'worker' ]; then
         NODE="${CONFIGURATION}-${WORKER}-${INDEX}"
+        CPU="$( get_cpu workerlist $INDEX )"
+        MEM="$( get_mem workerlist $INDEX )"
         DISK_LOCATION="$( get_disk workerlist $INDEX )"
     elif [ $IDENTITY == 'storage' ]; then
         NODE="${CONFIGURATION}-${STORAGE}-${INDEX}"
+        CPU="$( get_cpu storagelist $INDEX )"
+        MEM="$( get_mem storagelist $INDEX )"
         DISK_LOCATION="$( get_disk storagelist $INDEX )"
     fi
     mkdir -p "${DISK_LOCATION}"
@@ -217,7 +267,9 @@ create_node() {
     sudo qemu-img create -f qcow2 ${DISK_LOCATION}/${NODE}-1.img 200G
     cp ${IDENTITY}.xml ${DOMAIN_FILE}
     sed -i -e "
-        s,NAME,${NODE},;
+        s,NAME,${NODE},
+        s,%CPU%,${CPU},
+        s,%MEM%,${MEM},
         s,DISK0,${DISK_LOCATION}/${NODE}-0.img,;
         s,DISK1,${DISK_LOCATION}/${NODE}-1.img,
         s,%BR1%,${BRIDGE_INTERFACE}1,

libvirt/readconfig.sh

@@ -20,3 +20,8 @@ export WORKER_NODES_NUMBER="$( ${GET_CFG} worker_nodes_num )"
 export STORAGE_NODES_NUMBER="$( ${GET_CFG} storage_nodes_num )"
 export DOMAIN_DIRECTORY="$( ${GET_CFG} domain_dir )"
 export DEFAULT_DISK_DIR="$( ${GET_CFG} default_disk )"
+export DEFAULT_CPU="$( ${GET_CFG} default_cpu )"
+export DEFAULT_MEM="$( ${GET_CFG} default_mem )"
+export AIO_DEFAULT_CPU="$( ${GET_CFG} aio_default_cpu )"
+export AIO_DEFAULT_MEM="$( ${GET_CFG} aio_default_mem )"
+

libvirt/set_defaults.sh

@@ -21,5 +21,11 @@ export STORAGE=${STORAGE:-storage}
 export WORKER_NODES_NUMBER=${WORKER_NODES_NUMBER:-1}
 export STORAGE_NODES_NUMBER=${STORAGE_NODES_NUMBER:-1}
 
+export DEFAULT_CPU=${DEFAULT_CPU:-4}
+export DEFAULT_MEM=${DEFAULT_MEM:-16}
+
+export AIO_DEFAULT_CPU=${AIO_DEFAULT_CPU:-6}
+export AIO_DEFAULT_MEM=${AIO_DEFAULT_MEM:-18}
+
 export DOMAIN_DIRECTORY=${DOMAIN_DIRECTORY:-vms}
 export DEFAULT_DISK_DIR=${DEFAULT_DISK_DIR:-/var/lib/libvirt/images}

libvirt/storage.xml

@@ -1,8 +1,8 @@
 <domain type='kvm' id='187'>
   <name>NAME</name>
-  <memory unit='KiB'>16777216</memory>
+  <memory unit='GiB'>%MEM%</memory>
-  <currentMemory unit='KiB'>16777216</currentMemory>
+  <currentMemory unit='GiB'>%MEM%</currentMemory>
-  <vcpu placement='static'>4</vcpu>
+  <vcpu placement='static'>%CPU%</vcpu>
   <resource>
     <partition>/machine</partition>
   </resource>
@@ -16,7 +16,7 @@
   </features>
   <cpu match='exact'>
     <model fallback='forbid'>Nehalem</model>
-    <topology sockets='1' cores='4' threads='1'/>
+    <topology sockets='1' cores='%CPU%' threads='1'/>
     <feature policy='require' name='vmx'/>
     <feature policy='optional' name='svm'/>
   </cpu>

libvirt/worker.xml

@@ -1,8 +1,8 @@
 <domain type='kvm' id='187'>
   <name>NAME</name>
-  <memory unit='KiB'>16777216</memory>
+  <memory unit='GiB'>%MEM%</memory>
-  <currentMemory unit='KiB'>16777216</currentMemory>
+  <currentMemory unit='GiB'>%MEM%</currentMemory>
-  <vcpu placement='static'>4</vcpu>
+  <vcpu placement='static'>%CPU%</vcpu>
   <resource>
     <partition>/machine</partition>
   </resource>
@@ -16,7 +16,7 @@
   </features>
   <cpu match='exact'>
     <model fallback='forbid'>Nehalem</model>
-    <topology sockets='1' cores='4' threads='1'/>
+    <topology sockets='1' cores='%CPU%' threads='1'/>
     <feature policy='require' name='vmx'/>
     <feature policy='optional' name='svm'/>
   </cpu>