Merge "Review lab-setup files"

2023-08-11 13:50:12 +00:00 · 2023-08-11 13:50:12 +00:00 · dcef4d4bf9
commit dcef4d4bf9
parent 02f67a5746 0389ebc90a
24 changed files with 1400 additions and 853 deletions
--- a/tox.ini
+++ b/tox.ini
@ -25,7 +25,7 @@ commands =
         -not -name \*~                       \
         -not -name \*.md                     \
         -name \*.sh                          \
-         -print0 | xargs -0 bashate -v -iE006,E040"
+         -print0 | xargs -0 bashate -v -iE006,E040,E042"

 [flake8]
 ignore =
--- a/virtualbox/pybox/.gitignore
+++ b/virtualbox/pybox/.gitignore
@ -0,0 +1,6 @@
+localhost.yml
+__pycache__/
+consts/__pycache__/
+helper/__pycache__/
+utils/__pycache__/
+venv/
--- a/virtualbox/pybox/Parser.py
+++ b/virtualbox/pybox/Parser.py
@ -134,11 +134,17 @@ def parse_setup_config(parser: ArgumentParser):
                        default="sysadmin")
    parser.add_argument("--password", help=
                        """
-                        Password.
+                        admin password
                        """,
                        type=validate_password,
                        required=True)
-
+    parser.add_argument("--sysadmin-password", help=
+                        """
+                        sysadmin password
+                        This argument is optional
+                        The default value is the admin password
+                        """,
+                        type=validate_password)

 def parse_config_location(parser: ArgumentParser):
    """
@ -283,6 +289,16 @@ def parse_networking(parser: ArgumentParser):
                        installed.
                        """,
                        type=str)
+    parser.add_argument("--nat-controller-floating-ssh-port", help=
+                        """
+                        When oam network is configured as 'nat' a port on
+                        the vbox host is used for connecting to ssh on
+                        active controller.  No default value is configued. This
+                        is mandatory if --vboxnet-type is 'nat' for non
+                        AIO-SX deployments or if second controller is
+                        installed.
+                        """,
+                        type=str)
    parser.add_argument("--horizon-dashboard-port", help=
                        """
                        Port for the visualization of the StarlingX
--- a/virtualbox/pybox/README.md
+++ b/virtualbox/pybox/README.md
@ -38,27 +38,24 @@ Example stages:
 - ansible-controller-config updated based on args options.
 - rsync-config         # Rsync all files from --config-files-dir and
                         --config-files-dir* to /home/sysadmin.
- lab-setup1           # Run lab_setup with one or more --lab-setup-conf
+- setup-controller-0   # Run lab_setup with one or more --lab-setup-conf
                         files from controller-0.
 - unlock-controller-0  # Unlock controller-0 and wait for it to reboot.
- lab-setup2           # Run lab_setup with one or more --lab-setup-conf
-                         files from controller-0.

 Example chains: [create-lab, install-controller-0, config-controller,
-rsync-config, lab-setup1, unlock-controller-0, lab-setup2]. This chain
-will install an AIO-SX.
+setup-controller-0, unlock-controller-0]. This chain will install an AIO-SX.

 The autoinstaller has a predefined set of chains. The user can select from
 these chains and choose from which stage to which stage to do the install.
-For example, if the user already executed config_controller, they can choose
-to continue from rsync-config to lab-setup2.
+For example, if the user already executed config_controller, he or she can
+choose to continue from setup-controller-0 to unlock-controller-0.

 The user can also create a custom set of chains, as he sees fit by
 specifying them in the desired order. This allows better customization of
 the install process. For example, the user might want to execute his own
 script after config_controller.  In this case, he will have to specify a
 chain like this: [create-lab, install-controller-0, config-controller,
-rsync-config, custom-script1, lab-setup1, unlock-controller-0, lab-setup2]
+rsync-config, custom-script1, setup-controller-0, unlock-controller-0]

 The installer supports creating virtualbox snapshots after each stage so
 the user does not need to reinstall from scratch. The user can restore the
@ -125,10 +122,10 @@ will be configured and used.
    sudo apt install virtualbox socat git rsync sshpass openssh-client python3-pip python3-venv
    ```

-2. Create a NAT Network with the `VBoxManage` CLI that is installed with VirtualBox:
+2. Create a NAT Network with the `vboxmanage` CLI that is installed with VirtualBox:

    ```shell
-    VBoxManage natnetwork add --netname NatNetwork --network 10.10.10.0/24 --dhcp off --ipv6 on
+    vboxmanage natnetwork add --netname NatNetwork --network 10.10.10.0/24 --dhcp off --ipv6 on
    ```

 3. Checkout the repository, and set up Python's Virtual Environment with:
@ -172,6 +169,9 @@ running it):
      --snapshot
    ```

+The StarlingX admin and sysadmin passwords are administrated by the argument --password.
+Optionally, a distinct sysadmin password may be assigned by the argument --sysadmin-password.
+
 The script takes a while to do all the things (from creating a VM and
 installing an OS in it to configuring StarlingX). Several restarts might
 occur on the VM, and you might see a VirtualBox window with a prompt.
@ -184,4 +184,3 @@ running.
 ├── consts: contains modules for managing virtual lab environments, including classes for Lab, Subnets, NICs, OAM, Serial, nodes, and HostTimeout.
 ├── helper: contains modules for interacting with a StarlingX controller-0 server via a serial connection, configuring system settings, and managing virtual machines using VirtualBox.
 └── utils: contains modules for initializing logging, tracking and reporting KPIs, connecting and communicating with remote hosts via local domain socket, and sending files and directories to remote servers using rsync and paramiko libraries.
-
--- a/virtualbox/pybox/config/ansibleFiles/duplex_localhost.yml
+++ b/virtualbox/pybox/config/ansibleFiles/duplex_localhost.yml
@ -11,5 +11,8 @@ external_oam_node_0_address: 10.10.10.4
 external_oam_node_1_address: 10.10.10.5

 admin_username: admin
-admin_password: Li69nux*
-ansible_become_pass: <system-password>
+
+# The following password fields are overriden by the Automated Installer.
+# Refer to the README for instructions on how to administrate the StarlingX passwords.
+admin_password: <password>
+ansible_become_pass: <sysadmin-password>
--- a/virtualbox/pybox/config/ansibleFiles/simplex_localhost.yml
+++ b/virtualbox/pybox/config/ansibleFiles/simplex_localhost.yml
@ -9,5 +9,8 @@ external_oam_gateway_address: 10.10.10.1
 external_oam_floating_address: 10.10.10.3

 admin_username: admin
-admin_password: Li69nux*
-ansible_become_pass: <system-password>
+
+# The following password fields are overriden by the Automated Installer.
+# Refer to the README for instructions on how to administrate the StarlingX passwords.
+admin_password: <password>
+ansible_become_pass: <sysadmin-password>
--- a/virtualbox/pybox/config/labSetupFiles/lab_setup.conf
+++ b/virtualbox/pybox/config/labSetupFiles/lab_setup.conf
@ -32,7 +32,10 @@ INTERNALPNET="vlan|data0"
 DATA_INTERFACES="ethernet|eth1000|${DATAMTU}|data0 \
 ethernet|eth1001|${DATAMTU}|data1"

-OAM_INTERFACES="ethernet|enp0s3|1500|none"
+# Virtual Box
+DEFAULT_IF0="enp0s3"
+
+OAM_INTERFACES="ethernet|${DEFAULT_IF0}|1500|none"

 ## IP address pools to support VXLAN provider networks.  Each compute node will
 ## get an address allocated from within the specified pools
--- a/virtualbox/pybox/config/labSetupFiles/lab_setup.sh
+++ b/virtualbox/pybox/config/labSetupFiles/lab_setup.sh
@ -12,9 +12,15 @@ CLEAR_CHAIN="no"
 SYSTEM_NAME=""
 RAM_QUOTA=""

-DEFAULT_IF0=eth0
-DEFAULT_IF1=eth1
-DEFAULT_IF2=eth2
+# Bare Metal
+# DEFAULT_IF0="eth0"
+# DEFAULT_IF1="eth1"
+# DEFAULT_IF2="eth2"
+
+# Virtual Box
+DEFAULT_IF0="enp0s3"
+DEFAULT_IF1="enp0s8"
+DEFAULT_IF2="enp0s9"

 CLI_NOWRAP=--nowrap
 DEFAULT_OPENSTACK_PASSWORD="Li69nux*"
@ -101,7 +107,7 @@ SYSTEM_MODE=${system_mode:-none}
 DISTRIBUTED_CLOUD_ROLE="none"

 ## vswitch type
-VSWITCH_TYPE="avs"
+VSWITCH_TYPE="ovs-dpdk"

 ## Cinder's backends.
 # LVM, Ceph, both or none. If CONFIGURE_STORAGE_LVM is set, then Cinder will be configured by default
--- a/virtualbox/pybox/config/labSetupFiles/lab_setup1.sh
+++ b/virtualbox/pybox/config/labSetupFiles/lab_setup1.sh
@ -1,107 +0,0 @@
-#!/bin/bash
-
-## This file makes the necessary configuration for the unlock of the Controller-0
-
-DATE_FORMAT="%Y-%m-%d %T"
-LOG_FILE=${LOG_FILE:-"${HOME}/lab_setup_1.log"}
-VERBOSE_LEVEL=0
-##For now ceph_storage variable will be set to true but can be changed before executing the script
-CEPH_STORAGE="true"
-
-#Identify setup type
-SETUP_TYPE=$(system show | grep 'system_mode' | awk '{print $4}')
-
-OPENRC=/etc/platform/openrc
-source ${OPENRC}
-
-
-function info {
-    local MSG="$1"
-
-    echo ${MSG}
-    echo $(date +"${DATE_FORMAT}") ${MSG} >> ${LOG_FILE}
-}
-
-
-function log_command {
-    local CMD=$1
-    local MSG="[${OS_USERNAME}@${OS_PROJECT_NAME}]> RUNNING: ${CMD}"
-
-    set +e
-    if [ ${VERBOSE_LEVEL} -gt 0 ]; then
-        echo ${MSG}
-    fi
-    echo $(date +"${DATE_FORMAT}") ${MSG} >> ${LOG_FILE}
-
-    if [ ${VERBOSE_LEVEL} -gt 1 ]; then
-        eval ${CMD} 2>&1 | tee -a ${LOG_FILE}
-        RET=${PIPESTATUS[0]}
-    else
-        eval ${CMD} &>> ${LOG_FILE}
-        RET=$?
-    fi
-
-    if [ ${RET} -ne 0 ]; then
-        info "COMMAND FAILED (rc=${RET}): ${CMD}"
-        info "==========================="
-        info "Check \"${LOG_FILE}\" for more details, fix the issues and"
-        info "re-run the failed command manually."
-        exit 1
-    fi
-    set -e
-
-    return ${RET}
-}
-
-
-## Set OAM interface
-function configure_OAM_interface {
-
-    if [ "$SETUP_TYPE" == "simplex" ]; then
-        #Set OAM_IF variable
-        log_command "OAM_IF=enp0s3"
-        #Associate OAM_IF with Controller-0
-        log_command "system host-if-modify controller-0 $OAM_IF -c platform"
-        log_command "system interface-network-assign controller-0 $OAM_IF oam"
-
-    else
-        #Set Variables
-        log_command "OAM_IF=enp0s3 && MGMT_IF=enp0s8"
-
-        log_command "system host-if-modify controller-0 lo -c none"
-        local IFNET_UUIDS=$(system interface-network-list controller-0 | awk '{if ($6=="lo") print $4;}')
-        for UUID in $IFNET_UUIDS; do
-            log_command "system interface-network-remove ${UUID}"
-        done
-
-        #Associate variables with Controller-0
-        log_command "system host-if-modify controller-0 $OAM_IF -c platform"
-        log_command "system interface-network-assign controller-0 $OAM_IF oam"
-        log_command "system host-if-modify controller-0 $MGMT_IF -c platform"
-        log_command "system interface-network-assign controller-0 $MGMT_IF mgmt"
-        log_command "system interface-network-assign controller-0 $MGMT_IF cluster-host"
-    fi
-
-    return 0
-}
-
-
-## Initialize and set ceph_storage
-function initialize_ceph_storage {
-    echo "Setting host-based Ceph storage backend solution"
-
-    #Adding ceph backend
-    log_command "system storage-backend-add ceph --confirmed"
-
-    #Adding OSD on controller-0
-    log_command "system host-disk-list controller-0"
-    log_command "system host-disk-list controller-0 | awk '/\/dev\/sdb/{print \$2}' | xargs -i system host-stor-add controller-0 {}"
-    log_command "system host-stor-list controller-0"
-}
-
-
-configure_OAM_interface
-
-if [ "${CEPH_STORAGE}" == "true" ]; then
-    initialize_ceph_storage
-fi
--- a/virtualbox/pybox/config/labSetupFiles/lab_setup2.sh
+++ b/virtualbox/pybox/config/labSetupFiles/lab_setup2.sh
@ -1,83 +0,0 @@
-#!/bin/bash
-
-## This file makes the necessary configuration for the unlock of the Controller-1
-
-DATE_FORMAT="%Y-%m-%d %T"
-LOG_FILE=${LOG_FILE:-"${HOME}/lab_setup_2.log"}
-VERBOSE_LEVEL=0
-
-#Identify setup type
-SETUP_TYPE=$(system show | grep 'system_mode' | awk '{print $4}')
-
-OPENRC=/etc/platform/openrc
-source ${OPENRC}
-
-
-function info {
-    local MSG="$1"
-
-    echo ${MSG}
-    echo $(date +"${DATE_FORMAT}") ${MSG} >> ${LOG_FILE}
-}
-
-
-function log_command {
-    local CMD=$1
-    local MSG="[${OS_USERNAME}@${OS_PROJECT_NAME}]> RUNNING: ${CMD}"
-
-    set +e
-    if [ ${VERBOSE_LEVEL} -gt 0 ]; then
-        echo ${MSG}
-    fi
-    echo $(date +"${DATE_FORMAT}") ${MSG} >> ${LOG_FILE}
-
-    if [ ${VERBOSE_LEVEL} -gt 1 ]; then
-        eval ${CMD} 2>&1 | tee -a ${LOG_FILE}
-        RET=${PIPESTATUS[0]}
-    else
-        eval ${CMD} &>> ${LOG_FILE}
-        RET=$?
-    fi
-
-    if [ ${RET} -ne 0 ]; then
-        info "COMMAND FAILED (rc=${RET}): ${CMD}"
-        info "==========================="
-        info "Check \"${LOG_FILE}\" for more details, fix the issues and"
-        info "re-run the failed command manually."
-        exit 1
-    fi
-    set -e
-
-    return ${RET}
-}
-
-function configure_OAM_MGMT_interfaces {
-    #Set OAM_IF variable
-    log_command "OAM_IF=enp0s3"
-    #Associate OAM_IF with Controller-0
-    log_command "system host-if-modify controller-1 $OAM_IF -c platform"
-    log_command "system interface-network-assign controller-1 $OAM_IF oam"
-    log_command "system interface-network-assign controller-1 mgmt0 cluster-host"
-}
-
-
-##Configure ceph storage in controller-1
-function configure_ceph_storage {
-    echo "Setting host-based Ceph storage backend solution"
-    local CEPH=$(system storage-backend-list | grep 'ceph')
-
-    if [ -z "$CEPH" ]; then
-        echo "Ceph storage not set in controller-0, skipping process in controller-1"
-    else
-        #Adding OSD on controller-1
-        log_command "system host-disk-list controller-1"
-        log_command "system host-disk-list controller-1 | awk '/\/dev\/sdb/{print \$2}' | xargs -i system host-stor-add controller-1 {}"
-        log_command "system host-stor-list controller-1"
-    fi
-}
-
-configure_OAM_MGMT_interfaces
-
-configure_ceph_storage
-
-
--- a/virtualbox/pybox/consts/networking.py
+++ b/virtualbox/pybox/consts/networking.py
@ -166,15 +166,23 @@ class NICs:


 class OAM:
-    """The `OAM` class contains an IP address and netmask for the out-of-band
-    management (OAM) network."""
+    """The `OAM` class defines the out-of-band management (OAM) network."""

    OAM = {
+        "device": "enp0s3",
        "ip": "10.10.10.254",
        "netmask": "255.255.255.0",
    }


+class MGMT:
+    """The `MGMT` class defines the internal management (MGMT) network."""
+
+    MGMT = {
+        "device": "enp0s8",
+    }
+
+
 class Serial:
    """The `Serial` class contains configurations for the serial ports."""

--- a/virtualbox/pybox/consts/timeout.py
+++ b/virtualbox/pybox/consts/timeout.py
@ -19,4 +19,5 @@ class HostTimeout: #pylint: disable=too-few-public-methods
    HOST_INSTALL = 3600
    LAB_CONFIG = 5400
    INSTALL_PATCHES = 900
-    NETWORKING_OPERATIONAL = 60
+    NORMAL_OP = 90
+    REATTEMPT_DELAY = [0, 2, 5, 10, 30, 60, 2*60, 3*60, 5*60, 10*60]
--- a/virtualbox/pybox/helper/host_helper.py
+++ b/virtualbox/pybox/helper/host_helper.py
@ -10,7 +10,6 @@ locking, rebooting, and installing a host. The module uses streamexpect library
 facilitate stream parsing.
 """

-import time
 import streamexpect
 from consts.timeout import HostTimeout
 from utils import serial
@ -28,14 +27,17 @@ def unlock_host(stream, hostname):
        - Unlock host
    """

-    LOG.info("#### Unlock %s", hostname)
-    serial.send_bytes(stream, f"system host-list | grep {hostname}", expect_prompt=False)
+    cmd = f"system host-list | grep {hostname}"
+    serial.send_bytes(stream, cmd, expect_prompt=False)
    try:
        serial.expect_bytes(stream, "locked")
    except streamexpect.ExpectTimeout:
        LOG.info("Host %s not locked", hostname)
        return 1
-    serial.send_bytes(stream, f"system host-unlock {hostname}", expect_prompt=False)
+
+    LOG.info("#### Unlock %s", hostname)
+    cmd = f"system host-unlock {hostname}"
+    serial.send_bytes(stream, cmd, expect_prompt=False)
    LOG.info("Unlocking %s", hostname)
    return None

@ -51,14 +53,17 @@ def lock_host(stream, hostname):
        - Lock host
    """

-    LOG.info("Lock %s", hostname)
-    serial.send_bytes(stream, f"system host-list |grep {hostname}", expect_prompt=False)
+    cmd = f"system host-list |grep {hostname}"
+    serial.send_bytes(stream, cmd, expect_prompt=False)
    try:
        serial.expect_bytes(stream, "unlocked")
    except streamexpect.ExpectTimeout:
        LOG.info("Host %s not unlocked", hostname)
        return 1
-    serial.send_bytes(stream, f"system host-lock {hostname}", expect_prompt="keystone")
+
+    LOG.info("Lock %s", hostname)
+    cmd = f"system host-lock {hostname}"
+    serial.send_bytes(stream, cmd, expect_prompt="keystone")
    LOG.info("Locking %s", hostname)
    return None

@ -72,7 +77,8 @@ def reboot_host(stream, hostname):
    """

    LOG.info("Rebooting %s", hostname)
-    serial.send_bytes(stream, f"system host-reboot {hostname}", expect_prompt=False)
+    cmd = f"system host-reboot {hostname}"
+    serial.send_bytes(stream, cmd, expect_prompt=False)
    serial.expect_bytes(stream, "rebooting", HostTimeout.REBOOT)


@ -86,21 +92,16 @@ def install_host(stream, hostname, host_type, host_id):
        host_id(int): id to identify host
    """

-    time.sleep(10)
    LOG.info("Installing %s with id %s", hostname, host_id)
    if host_type == 'controller':
-        serial.send_bytes(stream,
-                          f"system host-update {host_id} personality=controller",
-                          expect_prompt=False)
+        cmd = f"system host-update {host_id} personality=controller"
+        serial.send_bytes(stream, cmd, expect_prompt=False)
    elif host_type == 'storage':
-        serial.send_bytes(stream,
-                          f"system host-update {host_id} personality=storage",
-                          expect_prompt=False)
+        cmd = f"system host-update {host_id} personality=storage"
+        serial.send_bytes(stream, cmd, expect_prompt=False)
    else:
-        serial.send_bytes(stream,
-                          f"system host-update {host_id} personality=compute hostname={hostname}",
-                          expect_prompt=False)
-    time.sleep(30)
+        cmd = f"system host-update {host_id} personality=compute hostname={hostname}"
+        serial.send_bytes(stream, cmd, expect_prompt=False)


 def disable_logout(stream):
@ -111,7 +112,8 @@ def disable_logout(stream):
    """

    LOG.info('Disabling automatic logout')
-    serial.send_bytes(stream, "export TMOUT=0")
+    cmd = "export TMOUT=0"
+    serial.send_bytes(stream, cmd)


 def change_password(stream, username, password):
--- a/virtualbox/pybox/helper/install_lab.py
+++ b/virtualbox/pybox/helper/install_lab.py
@ -7,8 +7,12 @@
 Contains helper functions that will configure basic system settings.
 """

+import subprocess
+import sys
+import time
+
 from consts.timeout import HostTimeout
-from utils import serial
+from utils import kpi, serial
 from utils.install_log import LOG
 from helper import host_helper

@ -19,13 +23,9 @@ def update_platform_cpus(stream, hostname, cpu_num=5):
    """

    LOG.info("Allocating %s CPUs for use by the %s platform.", cpu_num, hostname)
-    serial.send_bytes(
-        stream,
-        "\nsource /etc/platform/openrc; system host-cpu-modify " 
-        f"{hostname} -f platform -p0 {cpu_num}",
-        prompt="keystone",
-        timeout=300,
-    )
+    cmd = "\nsource /etc/platform/openrc;" \
+        f" system host-cpu-modify {hostname} -f platform -p0 {cpu_num}"
+    serial.send_bytes(stream, cmd, prompt="keystone", timeout=300)


 def set_dns(stream, dns_ip):
@ -34,12 +34,8 @@ def set_dns(stream, dns_ip):
    """

    LOG.info("Configuring DNS to %s.", dns_ip)
-    serial.send_bytes(
-        stream,
-        "source /etc/platform/openrc; system dns-modify "
-        f"nameservers={dns_ip}",
-        prompt="keystone",
-    )
+    cmd = f"source /etc/platform/openrc; system dns-modify nameservers={dns_ip}"
+    serial.send_bytes(stream, cmd, prompt="keystone")


 def config_controller(stream, password):
@ -47,13 +43,122 @@ def config_controller(stream, password):
    Configure controller-0 using optional arguments
    """

-    serial.send_bytes(
-        stream,
-        "ansible-playbook /usr/share/ansible/stx-ansible/playbooks/bootstrap.yml",
-        expect_prompt=False,
-    )
+    LOG.info("Executing the bootstrap ansible playbook")
+    cmd = "ansible-playbook /usr/share/ansible/stx-ansible/playbooks/bootstrap.yml"
+    serial.send_bytes(stream, cmd, expect_prompt=False)
    host_helper.check_password(stream, password=password)
-    ret = serial.expect_bytes(stream, "~$", timeout=HostTimeout.LAB_CONFIG)
+    serial.expect_bytes(stream, "~$", timeout=HostTimeout.LAB_CONFIG)
+
+    cmd = "echo [$?]"
+    serial.send_bytes(stream, cmd, expect_prompt=False, log=False)
+    ret = serial.expect_bytes(stream, "[0]", timeout=HostTimeout.NORMAL_OP, log=False)
    if ret != 0:
        LOG.info("Configuration failed. Exiting installer.")
-        raise Exception("Configcontroller failed")  # pylint: disable=E0012, W0719
+        raise SystemExit("Configcontroller failed")
+    LOG.info("Successful bootstrap ansible playbook execution")
+
+
+def fault_tolerant(scale=1):
+    """
+    Provides the scale argument to the fault-tolerant decorator.
+
+    Args:
+    - scale: re-attempt delay vector scale factor
+
+    Returns:
+    - fault-tolerant decorator.
+    """
+
+    def fault_tolerant_decorator(func):
+        """
+        Decorator to run a command in a fault-tolerant fashion.
+
+        Args:
+        - func: The function to be decorated.
+
+        Returns:
+        - fault-tolerant wrapper
+        """
+
+        def fault_tolerant_wrapper(*args, **kwargs):
+
+            """
+            Runs a command in a fault-tolerant fashion.
+
+            The function provides a recovery mechanism with progressive re-attempt delays
+            The first attempt is the normal command execution. If the command fails, the first
+            re-attempt runs after 2s, and the re-attempt delay goes increasing until 10 min.
+            The last re-attempts, with longer delays are intended to help the user to
+            salvage the ongoing installation, if the system does not recover automatically.
+
+            Intentionally, the function does not provide a return code, due to the following
+            reason. To ensure system integrity, the function stops the program execution,
+            if it can not achieve a successful result, after a maximum number of retries.
+
+            Hence, any subsequent functions may safely rely on the system integrity.
+
+            Args:
+            - cmd: The command to be executed.
+
+            Returns: None
+            """
+
+            delay = HostTimeout.REATTEMPT_DELAY
+            reattempt_delay = scale*delay
+            max_attempts = len(reattempt_delay)
+            attempt = 1
+            while True:
+                cmd = kwargs['cmd']
+                try:
+                    return_code = func(*args, **kwargs)
+                    assert return_code == 0
+                    break
+                except AssertionError as exc:
+                    if attempt < max_attempts:
+                        LOG.warning(
+                            "#### Failed command:\n$ %s [attempt: %s/%s]\n",
+                            cmd, attempt, max_attempts
+                        )
+                        LOG.info(
+                            "Trying again after %s ... ",
+                            kpi.get_formated_time(reattempt_delay[attempt])
+                        )
+                        time.sleep(reattempt_delay[attempt])
+                        attempt = attempt + 1
+                    else:
+                        LOG.error(
+                            "#### Failed command:\n$ %s [attempt: %s/%s]\n",
+                            cmd, attempt, max_attempts
+                        )
+                        raise TimeoutError from exc
+                except Exception as exc:  # pylint: disable=broad-except
+                    LOG.error(
+                        "#### Failed command:\n$ %s\nError: %s",
+                        cmd, repr(exc)
+                    )
+                    sys.exit(1)
+
+        return fault_tolerant_wrapper
+
+    return fault_tolerant_decorator
+
+
+def exec_cmd(cmd):
+
+    """
+    Execute a local command on the host machine in a fault-tolerant fashion.
+    Refer to the fault_tolerant decorator for more details.
+    """
+
+    @fault_tolerant()
+    def exec_cmd_ft(*args, **kwargs):  # pylint: disable=unused-argument
+
+        LOG.info("#### Executing command on the host machine:\n$ %s\n", cmd)
+        with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process:
+            for line in iter(process.stdout.readline, b''):
+                LOG.info("%s", line.decode("utf-8").strip())
+            process.wait()
+        return process.returncode
+
+
+    exec_cmd_ft(**{'cmd': cmd})
--- a/virtualbox/pybox/helper/tests/test_install_lab.py
+++ b/virtualbox/pybox/helper/tests/test_install_lab.py
@ -3,9 +3,238 @@ Unit tests related to install_lab
 """

 import unittest
-from unittest.mock import MagicMock, patch
 import install_lab

+from consts.timeout import HostTimeout
+from helper.install_lab import exec_cmd
+from unittest.mock import MagicMock, patch, call
+from utils import kpi
+
+
+class ExecCmdTestCase(unittest.TestCase):
+    """
+    Class to test help function exec_cmd
+    """
+
+    SUCCESS = 0
+    FAILED1 = 1
+
+    def setUp(self):
+        """
+        Method to set up the parameters used on the tests in this class
+        """
+        self.cmd = 'print ("Hello!")'
+        self.reattempt_delay = HostTimeout.REATTEMPT_DELAY
+        self.max_attempts = len(self.reattempt_delay)
+        self.counter = 0
+        self.trigger = 0
+        self.result = None
+
+
+    def dynamicMock(self, *args, **kwargs):
+        """
+        Method to simulate a function with temporary failures
+        """
+        process = MagicMock()
+        process.__enter__.return_value = process
+
+        self.counter += 1
+
+        if self.counter == self.trigger:
+            self.counter = 0
+            attrs = {
+                'wait.return_value': None,
+                'stdout.readline.side_effect': [bytes('Hello!\n', 'utf-8'), b'']
+            }
+            process.configure_mock(**attrs)
+            process.returncode = self.SUCCESS
+        else:
+            attrs = {
+                'wait.return_value': None,
+                'stdout.readline.side_effect': [b'']
+            }
+            process.configure_mock(**attrs)
+            process.returncode = self.FAILED1
+
+        return process
+
+
+    @patch('time.sleep')
+    @patch('utils.install_log.LOG.error')
+    @patch('utils.install_log.LOG.warning')
+    @patch('utils.install_log.LOG.info')
+    @patch('subprocess.Popen')
+    def test_exec_cmd_1st_attempt_ok(
+            self, m_s_Popen, m_LOG_info, m_LOG_warning, m_LOG_error, m_time_sleep
+        ):
+        """
+        Test successful call in the first attempt (normal case)
+        This test focus on info messages. No warnings or error messages are expected.
+        The expect result code is None (refer to the description of fault_tolerant decorator)
+        """
+        # Setup
+        m_time_sleep.return_value = 0
+        process = MagicMock()
+        attrs = {
+            'wait.return_value': None,
+            'returncode': self.SUCCESS,
+            'stdout.readline.side_effect': [bytes('Hello!\n', 'utf-8'), b'']
+        }
+        process.configure_mock(**attrs)
+        m_s_Popen.return_value.__enter__.return_value = process
+
+        # Run
+        self.result = exec_cmd(self.cmd)
+
+        # Assert
+        calls = [
+            call("#### Executing command on the host machine:\n$ %s\n", self.cmd),
+            call("%s", "Hello!"),
+        ]
+        m_LOG_info.assert_has_calls(calls, any_order=False)
+        self.assertEqual(m_LOG_warning.call_count, 0)
+        self.assertEqual(m_LOG_error.call_count, 0)
+        self.assertIsNone(self.result)
+
+
+    @patch('time.sleep')
+    @patch('utils.install_log.LOG.error')
+    @patch('utils.install_log.LOG.warning')
+    @patch('utils.install_log.LOG.info')
+    @patch('subprocess.Popen')
+    def test_exec_cmd_3rd_attempt_ok(
+            self, m_s_Popen, m_LOG_info, m_LOG_warning, m_LOG_error, m_time_sleep
+        ):
+        """
+        Test successful call after a few retries (this may occor in system instability scenarios)
+        This test focus on log info messages, for re-attempt scenarios.
+        Warning messages are covered in next test.
+        No error messages are expected.
+        The expect result code is None (refer to the description of fault_tolerant decorator)
+        """
+        # Setup
+        m_s_Popen.side_effect = self.dynamicMock
+        m_time_sleep.return_value = 0
+        self.trigger = 3
+
+        # Run
+        self.result = exec_cmd(self.cmd)
+
+        # Assert
+        calls = [
+            call('#### Executing command on the host machine:\n$ %s\n', self.cmd),
+            call("Trying again after %s ... ", kpi.get_formated_time(self.reattempt_delay[1])),
+
+            call('#### Executing command on the host machine:\n$ %s\n', self.cmd),
+            call("Trying again after %s ... ", kpi.get_formated_time(self.reattempt_delay[2])),
+
+            call("#### Executing command on the host machine:\n$ %s\n", self.cmd),
+            call("%s", "Hello!"),
+        ]
+        m_LOG_info.assert_has_calls(calls, any_order=False)
+        self.assertEqual(m_LOG_error.call_count, 0)
+        self.assertEqual(m_LOG_warning.call_count, 2)
+        self.assertIsNone(self.result)
+
+
+    @patch('time.sleep')
+    @patch('utils.install_log.LOG.error')
+    @patch('utils.install_log.LOG.warning')
+    @patch('subprocess.Popen')
+    def test_exec_cmd_5th_attempt_ok(
+            self, m_s_Popen, m_LOG_warning, m_LOG_error, m_time_sleep
+        ):
+        """
+        Test successful call after a few retries (thsi may occor in system instability scenarios)
+        This test focus on warning messages, for re-attempt scenarios.
+        No error messages are expected.
+        The expect result code is None (refer to the description of fault_tolerant decorator)
+        """
+        # Setup
+        m_s_Popen.side_effect = self.dynamicMock
+        m_time_sleep.return_value = 0
+        self.trigger = 5
+
+        # Run
+        self.result = exec_cmd(self.cmd)
+
+        # Assert
+        calls = [
+            call('#### Failed command:\n$ %s [attempt: %s/%s]\n', self.cmd, 1, 10),
+            call('#### Failed command:\n$ %s [attempt: %s/%s]\n', self.cmd, 2, 10),
+            call('#### Failed command:\n$ %s [attempt: %s/%s]\n', self.cmd, 3, 10),
+            call('#### Failed command:\n$ %s [attempt: %s/%s]\n', self.cmd, 4, 10),
+        ]
+        m_LOG_warning.assert_has_calls(calls, any_order=False)
+        self.assertEqual(m_LOG_error.call_count, 0)
+        self.assertIsNone(self.result)
+
+
+    @patch("sys.exit")
+    @patch('time.sleep')
+    @patch('utils.install_log.LOG.error')
+    @patch('utils.install_log.LOG.warning')
+    @patch('utils.install_log.LOG.info')
+    @patch('subprocess.Popen')
+    def test_exec_cmd_failed(
+            self, m_s_Popen, m_LOG_info, m_LOG_warning, m_LOG_error, m_time_sleep, mock_exit
+        ):
+        """
+        Test unsuccessful call
+        This may occur after a maximum number of retries, in strong system instability scenarios)
+        This test focus on info and error messages, for the failure scenarios.
+        The expected warning messages are the same as for successful calls, covered previously.
+        The expect result code is None (refer to the description of fault_tolerant decorator)
+        """
+
+        # Setup
+        m_s_Popen.side_effect = self.dynamicMock
+        m_LOG_warning.return_value = 0
+        m_time_sleep.return_value = 0
+        mock_exit.side_effect = SystemExit(1)
+        self.trigger = self.max_attempts+1
+
+        with self.assertRaises(TimeoutError):
+            # Run
+            self.result = exec_cmd(self.cmd)
+
+        # Assert
+        calls = [
+            call('#### Executing command on the host machine:\n$ %s\n', self.cmd),
+            call("Trying again after %s ... ", kpi.get_formated_time(self.reattempt_delay[1])),
+
+            call('#### Executing command on the host machine:\n$ %s\n', self.cmd),
+            call("Trying again after %s ... ", kpi.get_formated_time(self.reattempt_delay[2])),
+
+            call('#### Executing command on the host machine:\n$ %s\n', self.cmd),
+            call("Trying again after %s ... ", kpi.get_formated_time(self.reattempt_delay[3])),
+
+            call('#### Executing command on the host machine:\n$ %s\n', self.cmd),
+            call("Trying again after %s ... ", kpi.get_formated_time(self.reattempt_delay[4])),
+
+            call('#### Executing command on the host machine:\n$ %s\n', self.cmd),
+            call("Trying again after %s ... ", kpi.get_formated_time(self.reattempt_delay[5])),
+
+            call('#### Executing command on the host machine:\n$ %s\n', self.cmd),
+            call("Trying again after %s ... ", kpi.get_formated_time(self.reattempt_delay[6])),
+
+            call('#### Executing command on the host machine:\n$ %s\n', self.cmd),
+            call("Trying again after %s ... ", kpi.get_formated_time(self.reattempt_delay[7])),
+
+            call('#### Executing command on the host machine:\n$ %s\n', self.cmd),
+            call("Trying again after %s ... ", kpi.get_formated_time(self.reattempt_delay[8])),
+
+            call('#### Executing command on the host machine:\n$ %s\n', self.cmd),
+            call("Trying again after %s ... ", kpi.get_formated_time(self.reattempt_delay[9])),
+        ]
+        m_LOG_info.assert_has_calls(calls, any_order=False)
+
+        m_LOG_error.assert_called_once_with(
+            "#### Failed command:\n$ %s [attempt: %s/%s]\n",
+            self.cmd, self.max_attempts, self.max_attempts
+        )
+        self.assertIsNone(self.result)
+

 class UpdatePlatformCpusTestCase(unittest.TestCase):
    """
@ -89,12 +318,18 @@ class ConfigControllerTestCase(unittest.TestCase):
        install_lab.config_controller(self.mock_stream, password=self.mock_password)

        # Assert
-        mock_serial.send_bytes.assert_called_once_with(
-            self.mock_stream, self.command_string, expect_prompt=False
-        )
+        calls = [
+            call(self.mock_stream, self.command_string, expect_prompt=False),
+            call(self.mock_stream, 'echo [$?]', expect_prompt=False, log=False),
+        ]
+        mock_serial.send_bytes.assert_has_calls(calls, any_order=False)
+        calls = [
+            call(self.mock_stream, "~$", timeout=HostTimeout.LAB_CONFIG),
+            call(self.mock_stream, '[0]', timeout=HostTimeout.NORMAL_OP, log=False),
+        ]
+        mock_serial.expect_bytes.assert_has_calls(calls, any_order=False)
        mock_check_password.assert_called_once_with(self.mock_stream, password=self.mock_password)
-        mock_serial.expect_bytes.assert_called_once_with(self.mock_stream, "~$",
-                                                         timeout=install_lab.HostTimeout.LAB_CONFIG)
+

    @patch("install_lab.serial")
    @patch("install_lab.host_helper.check_password")
@ -107,16 +342,21 @@ class ConfigControllerTestCase(unittest.TestCase):
        mock_serial.expect_bytes.return_value = 1

        # Run
-        with self.assertRaises(Exception):
+        with self.assertRaises(SystemExit):
            install_lab.config_controller(self.mock_stream, password=self.mock_password)

        # Assert
-        mock_serial.send_bytes.assert_called_once_with(
-            self.mock_stream, self.command_string, expect_prompt=False
-        )
+        calls = [
+            call(self.mock_stream, self.command_string, expect_prompt=False),
+            call(self.mock_stream, 'echo [$?]', expect_prompt=False, log=False),
+        ]
+        mock_serial.send_bytes.assert_has_calls(calls, any_order=False)
+        calls = [
+            call(self.mock_stream, "~$", timeout=HostTimeout.LAB_CONFIG),
+            call(self.mock_stream, '[0]', timeout=HostTimeout.NORMAL_OP, log=False),
+        ]
+        mock_serial.expect_bytes.assert_has_calls(calls, any_order=False)
        mock_check_password.assert_called_once_with(self.mock_stream, password=self.mock_password)
-        mock_serial.expect_bytes.assert_called_once_with(self.mock_stream, "~$",
-                                                         timeout=install_lab.HostTimeout.LAB_CONFIG)


 if __name__ == '__main__':
--- a/virtualbox/pybox/helper/vboxmanage.py
+++ b/virtualbox/pybox/helper/vboxmanage.py
@ -22,10 +22,8 @@ def vboxmanage_version():
    Return version of vbox.
    """

-    version = subprocess.check_output(
-        ["vboxmanage", "--version"], stderr=subprocess.STDOUT
-    )
-
+    cmd = ["vboxmanage", "--version"]
+    version = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    return version


@ -40,17 +38,18 @@ def vboxmanage_extpack():

    LOG.info("Downloading extension pack")
    filename = f"Oracle_VM_VirtualBox_Extension_Pack-{version_path}.vbox-extpack"
-    cmd = f"http://download.virtualbox.org/virtualbox/{version_path}/{filename}"
-    result = subprocess.check_output(
-        ["wget", cmd, "-P", "/tmp"], stderr=subprocess.STDOUT
-    )
+    cmd = [
+        "wget",
+        f"http://download.virtualbox.org/virtualbox/{version_path}/{filename}",
+        "-P",
+        "/tmp"
+    ]
+    result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    LOG.info(result)

    LOG.info("Installing extension pack")
-    result = subprocess.check_output(
-        ["vboxmanage", "extpack", "install", "/tmp/" + filename, "--replace"],
-        stderr=subprocess.STDOUT,
-    )
+    cmd = ["vboxmanage", "extpack", "install", "/tmp/" + filename, "--replace"]
+    result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    LOG.info(result)


@ -60,7 +59,7 @@ def get_all_vms(labname, option="vms"):

    Args:
        labname (str): The name of the lab to which the VMs belong.
-        option (str, optional): The VBoxManage command option to use when listing VMs.
+        option (str, optional): The vboxmanage command option to use when listing VMs.
            Defaults to "vms".

    Returns:
@ -120,6 +119,7 @@ def take_snapshot(labname, snapshot_name):

    _resume_running_vms(runningvms)

+    LOG.info("Waiting 10s before running VMs")
    time.sleep(10)

    if runningvms:
@ -213,18 +213,22 @@ def restore_snapshot(node_list, name):
    LOG.info("Restore snapshot of %s for hosts %s", name, node_list)
    if len(node_list) != 0:
        vboxmanage_controlvms(node_list, "poweroff")
+        LOG.info("Waiting 5s")
        time.sleep(5)
    if len(node_list) != 0:
        for host in node_list:
            vboxmanage_restoresnapshot(host, name)
+            LOG.info("Waiting 5s")
            time.sleep(5)
        for host in node_list:
            if "controller-0" not in host:
                vboxmanage_startvm(host)
+                LOG.info("Waiting 10s")
                time.sleep(10)
        for host in node_list:
            if "controller-0" in host:
                vboxmanage_startvm(host)
+                LOG.info("Waiting 10s")
                time.sleep(10)


@ -233,9 +237,8 @@ def vboxmanage_list(option="vms"):
    This returns a list of vm names.
    """

-    result = subprocess.check_output(
-        ["vboxmanage", "list", option], stderr=subprocess.STDOUT
-    )
+    cmd = ["vboxmanage", "list", option]
+    result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    vms_list = []
    for item in result.splitlines():
        vm_name = re.match(b'"(.*?)"', item)
@ -251,10 +254,8 @@ def vboxmanage_showinfo(host):

    if not isinstance(host, str):
        host.decode("utf-8")
-    result = subprocess.check_output(
-        ["vboxmanage", "showvminfo", host, "--machinereadable"],
-        stderr=subprocess.STDOUT,
-    )
+    cmd = ["vboxmanage", "showvminfo", host, "--machinereadable"]
+    result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    return result


@ -267,20 +268,18 @@ def vboxmanage_createvm(hostname, labname):
    assert labname, "Labname is required"
    group = "/" + labname
    LOG.info("Creating VM %s", hostname)
-    subprocess.check_output(
-        [
-            "vboxmanage",
-            "createvm",
-            "--name",
-            hostname,
-            "--register",
-            "--ostype",
-            "Linux_64",
-            "--groups",
-            group,
-        ],
-        stderr=subprocess.STDOUT,
-    )
+    cmd = [
+        "vboxmanage",
+        "createvm",
+        "--name",
+        hostname,
+        "--register",
+        "--ostype",
+        "Linux_64",
+        "--groups",
+        group,
+    ]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)


 def vboxmanage_deletevms(hosts=None):
@ -293,10 +292,9 @@ def vboxmanage_deletevms(hosts=None):
    if len(hosts) != 0:
        for hostname in hosts:
            LOG.info("Deleting VM %s", hostname)
-            subprocess.check_output(
-                ["vboxmanage", "unregistervm", hostname, "--delete"],
-                stderr=subprocess.STDOUT,
-            )
+            cmd = ["vboxmanage", "unregistervm", hostname, "--delete"]
+            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+            LOG.info("Waiting 10s")
            time.sleep(10)
            # in case medium is still present after delete
            vboxmanage_deletemedium(hostname)
@ -318,25 +316,21 @@ def vboxmanage_hostonlyifcreate(name="vboxnet0", oam_ip=None, netmask=None):
    assert netmask, "Must provide an OAM Netmask"

    LOG.info("Creating Host-only Network")
-
-    subprocess.check_output(
-        ["vboxmanage", "hostonlyif", "create"], stderr=subprocess.STDOUT
-    )
+    cmd = ["vboxmanage", "hostonlyif", "create"]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)

    LOG.info("Provisioning %s with IP %s and Netmask %s", name, oam_ip, netmask)
-    subprocess.check_output(
-        [
-            "vboxmanage",
-            "hostonlyif",
-            "ipconfig",
-            name,
-            "--ip",
-            oam_ip,
-            "--netmask",
-            netmask,
-        ],
-        stderr=subprocess.STDOUT,
-    )
+    cmd = [
+        "vboxmanage",
+        "hostonlyif",
+        "ipconfig",
+        name,
+        "--ip",
+        oam_ip,
+        "--netmask",
+        netmask,
+    ]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)


 def vboxmanage_hostonlyifdelete(name="vboxnet0"):
@ -346,9 +340,8 @@ def vboxmanage_hostonlyifdelete(name="vboxnet0"):

    assert name, "Must provide network name"
    LOG.info("Removing Host-only Network")
-    subprocess.check_output(
-        ["vboxmanage", "hostonlyif", "remove", name], stderr=subprocess.STDOUT
-    )
+    cmd = ["vboxmanage", "hostonlyif", "remove", name]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)


 def vboxmanage_modifyvm(hostname, vm_config=None):
@ -397,9 +390,8 @@ def vboxmanage_modifyvm(hostname, vm_config=None):
    cmd.extend(["--boot4"])
    cmd.extend(["net"])

-    LOG.info(cmd)
-
-    LOG.info("Updating VM %s configuration", hostname)
+    LOG.info("#### Updating VM %s configuration", hostname)
+    LOG.info("#### Executing command on the host machine:\n$ %s\n", ' '.join(str(i) for i in cmd))
    subprocess.check_output(cmd, stderr=subprocess.STDOUT)


@ -525,21 +517,20 @@ def vboxmanage_storagectl(hostname=None, storectl="sata", hostiocache="off"):

    assert hostname, "Hostname is required"
    assert storectl, "Type of storage controller is required"
+
    LOG.info("Creating %s storage controller on VM %s", storectl, hostname)
-    subprocess.check_output(
-        [
-            "vboxmanage",
-            "storagectl",
-            hostname,
-            "--name",
-            storectl,
-            "--add",
-            storectl,
-            "--hostiocache",
-            hostiocache,
-        ],
-        stderr=subprocess.STDOUT,
-    )
+    cmd = [
+        "vboxmanage",
+        "storagectl",
+        hostname,
+        "--name",
+        storectl,
+        "--add",
+        storectl,
+        "--hostiocache",
+        hostiocache,
+    ]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)


 def vboxmanage_storageattach(hostname, storage_config):
@ -552,7 +543,7 @@ def vboxmanage_storageattach(hostname, storage_config):
            Possible key values: storectl, storetype, disk, port_num, device_num.

    Returns:
-        str: The output of the VBoxManage command.
+        str: The output of the vboxmanage command.
    """

    assert hostname, "Hostname is required"
@ -574,25 +565,22 @@ def vboxmanage_storageattach(hostname, storage_config):
        storectl,
        hostname,
    )
-
-    return subprocess.check_output(
-        [
-            "vboxmanage",
-            "storageattach",
-            hostname,
-            "--storagectl",
-            storectl,
-            "--medium",
-            disk,
-            "--type",
-            storetype,
-            "--port",
-            port_num,
-            "--device",
-            device_num,
-        ],
-        stderr=subprocess.STDOUT,
-    )
+    cmd = [
+        "vboxmanage",
+        "storageattach",
+        hostname,
+        "--storagectl",
+        storectl,
+        "--medium",
+        disk,
+        "--type",
+        storetype,
+        "--port",
+        port_num,
+        "--device",
+        device_num,
+    ]
+    return subprocess.check_output(cmd, stderr=subprocess.STDOUT)


 def vboxmanage_deletemedium(hostname, vbox_home_dir="/home"):
@ -625,20 +613,18 @@ def vboxmanage_deletemedium(hostname, vbox_home_dir="/home"):
    for disk in disk_list:
        LOG.info("Disconnecting disk %s from vbox.", disk)
        try:
-            result = subprocess.check_output(
-                [
-                    "vboxmanage",
-                    "closemedium",
-                    "disk",
-                    f"{vbox_home_dir}{disk}",
-                    "--delete",
-                ],
-                stderr=subprocess.STDOUT,
-            )
+            cmd = [
+                "vboxmanage",
+                "closemedium",
+                "disk",
+                f"{vbox_home_dir}{disk}",
+                "--delete",
+            ]
+            result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
            LOG.info(result)
        except subprocess.CalledProcessError as exception:
            # Continue if failures, disk may not be present
-            LOG.info(
+            LOG.warning(
                "Error disconnecting disk, continuing. "
                "Details: stdout: %s stderr: %s",
                exception.stdout,
@ -647,8 +633,8 @@ def vboxmanage_deletemedium(hostname, vbox_home_dir="/home"):
        LOG.info("Removing backing file %s", disk)
        try:
            os.remove(f"{vbox_home_dir}{disk}")
-        except: # pylint: disable=bare-except
-            pass
+        except Exception as exc:
+            LOG.debug("Failure at removing backing file\nError: %s\n", repr(exc))


 def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir="/home"):
@ -691,25 +677,23 @@ def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir="/home"
        )

        try:
-            result = subprocess.check_output(
-                [
-                    "vboxmanage",
-                    "createmedium",
-                    "disk",
-                    "--size",
-                    str(disk),
-                    "--filename",
-                    file_name,
-                    "--format",
-                    "vdi",
-                    "--variant",
-                    "standard",
-                ],
-                stderr=subprocess.STDOUT,
-            )
+            cmd = [
+                "vboxmanage",
+                "createmedium",
+                "disk",
+                "--size",
+                str(disk),
+                "--filename",
+                file_name,
+                "--format",
+                "vdi",
+                "--variant",
+                "standard",
+            ]
+            result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
            LOG.info(result)
        except subprocess.CalledProcessError as exception:
-            LOG.info("Error stdout: %s stderr: %s", exception.stdout, exception.stderr)
+            LOG.error("Error stdout: %s stderr: %s", exception.stdout, exception.stderr)
            raise
        vboxmanage_storageattach(
            hostname,
@ -723,6 +707,8 @@ def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir="/home"
        )
        disk_count += 1
        port_num += 1
+
+    LOG.info("Waiting 5s")
    time.sleep(5)


@ -747,9 +733,8 @@ def vboxmanage_startvm(hostname=None, headless=False, force=False):
        LOG.info("Host %s is already started", hostname)
    else:
        LOG.info("Powering on VM %s", hostname)
-        result = subprocess.check_output(
-            ["vboxmanage", "startvm", hostname, "--type", interface_type], stderr=subprocess.STDOUT
-        )
+        cmd = ["vboxmanage", "startvm", hostname, "--type", interface_type]
+        result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        LOG.info(result)

    # Wait for VM to start
@ -808,6 +793,8 @@ def vboxmanage_restoresnapshot(host=None, name=None):
    subprocess.call(
        ["vboxmanage", "snapshot", host, "restore", name], stderr=subprocess.STDOUT
    )
+
+    LOG.info("Waiting 10s")
    time.sleep(10)


@ -822,6 +809,7 @@ def vboxmanage_getrulename(network, local_port):
    Returns:
        (str): Name of rule or empty
    """
+
    # List information about all nat networks in VirtualBox
    cmd = ["vboxmanage", "list", "natnets"]
    result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
@ -861,7 +849,9 @@ def vboxmanage_addportforward(rule_name, local_port, guest_ip, guest_port, netwo
        True if the port was added
        False if an error occurred when trying to add the port-forward rule.
    """
+
    rule = f"{rule_name}:tcp:[]:{local_port}:[{guest_ip}]:{guest_port}"
+
    LOG.info("Creating port-forwarding rule to: %s", rule)
    cmd = [
        "vboxmanage",
@ -891,6 +881,7 @@ def vboxmanage_deleteportforward(rule_name, network):
    Returns:
        None
    """
+
    LOG.info(
        "Removing previous forwarding rule '%s' from NAT network '%s'",
        rule_name,
--- a/virtualbox/pybox/install_vbox.py
+++ b/virtualbox/pybox/install_vbox.py
--- a/virtualbox/pybox/tests/test_install_vbox.py
+++ b/virtualbox/pybox/tests/test_install_vbox.py
@ -1,7 +1,14 @@
-import unittest
-from unittest.mock import MagicMock, patch, call, ANY
 import install_vbox
+import unittest
+
+from consts.networking import OAM, MGMT
 from dataclasses import dataclass
+from unittest.mock import MagicMock, patch, call, ANY
+
+
+# Network
+OAM_CONFIG = [getattr(OAM, attr) for attr in dir(OAM) if not attr.startswith('__')]
+MGMT_CONFIG = [getattr(MGMT, attr) for attr in dir(MGMT) if not attr.startswith('__')]


@dataclass
@ -148,11 +155,12 @@ class SetupNetworkingTestCase(unittest.TestCase):
            call(self.mock_stream, password=self.mock_password),
            call(self.mock_stream, password=self.mock_password)
        ])
+        oam_if = OAM_CONFIG[0]['device']
        mock_serial.send_bytes.assert_any_call(self.mock_stream,
-                                               f"sudo /sbin/ip addr add {self.mock_ip}/24 dev enp0s3",
+                                               f"sudo /sbin/ip addr add {self.mock_ip}/24 dev {oam_if}",
                                               expect_prompt=False)
        mock_serial.send_bytes.assert_any_call(self.mock_stream,
-                                               "sudo /sbin/ip link set enp0s3 up",
+                                               f"sudo /sbin/ip link set {oam_if} up",
                                               expect_prompt=False)
        mock_serial.send_bytes.assert_any_call(self.mock_stream,
                                               f"sudo route add default gw {self.mock_gateway_ip}",
@ -203,13 +211,14 @@ class FixNetworkingTestCase(unittest.TestCase):
        install_vbox.fix_networking(self.mock_stream, self.mock_release_r3, self.mock_password)

        # Assert
+        oam_if = OAM_CONFIG[0]['device']
        mock_serial.send_bytes.assert_any_call(self.mock_stream,
-                                               "sudo /sbin/ip link set enp0s3 down",
+                                               f"sudo /sbin/ip link set {oam_if} down",
                                               expect_prompt=False)
        mock_host_helper.check_password.assert_called_with(self.mock_stream, password=self.mock_password)
        mock_serial.send_bytes.assert_any_call(
            self.mock_stream,
-            "sudo /sbin/ip link set enp0s3 up",
+            f"sudo /sbin/ip link set {oam_if} up",
            expect_prompt=False)
        mock_host_helper.check_password.assert_called_with(self.mock_stream, password=self.mock_password)

--- a/virtualbox/pybox/utils/install_log.py
+++ b/virtualbox/pybox/utils/install_log.py
@ -53,12 +53,13 @@ def init_logging(lab_name, log_path=None):
    # Create symbolic link to latest logs of this lab
    try:
        os.unlink(lab_log_path + "/latest")
-    except:  # pylint: disable=bare-except
+    except FileNotFoundError:
        pass
+
    os.symlink(LOG_DIR, lab_log_path + "/latest")


 def get_log_dir():
-    """This method returns the directory path of the current logging run."""
+    """This method returns the log directory"""

    return LOG_DIR
--- a/virtualbox/pybox/utils/serial.py
+++ b/virtualbox/pybox/utils/serial.py
@ -27,24 +27,27 @@ def connect(hostname, port=10000, prefix=""):

    if prefix:
        prefix = f"{prefix}_"
+
    socketname = f"/tmp/{prefix}{hostname}"
    if 'controller-0' in hostname:
        socketname += '_serial'
+
    LOG.info("Connecting to %s at %s", hostname, socketname)
-
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
-
    try:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
        sock.connect(('localhost', port))
-    except: # pylint: disable=bare-except
-        LOG.info("Connection failed")
-        pass  # pylint: disable=unnecessary-pass
-        # disconnect(sock)
-        sock = None
-    # TODO (WEI): double check this # pylint: disable=fixme
-    if sock:
-        sock.setblocking(False)
+
+        # TODO (WEI): double check this # pylint: disable=fixme
+        if sock:
+            sock.setblocking(False)
+
+    except Exception as exc:
+        LOG.info("Failed sock connection")
+        LOG.debug("Error:\n%s\n", repr(exc))
+        if sock:
+            sock.close()
+            sock = None

    return sock

@ -85,12 +88,12 @@ def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, fl
            trash = stream.poll(1)  # flush input buffers
            if trash:
                try:
-                    LOG.info("Buffer has bytes before cmd execution: %s",
+                    LOG.debug("Buffer has bytes before cmd execution: %s",
                             trash.decode('utf-8'))
-                except Exception:  # pylint: disable=W0703
-                    pass
-        except streamexpect.ExpectTimeout:
-            pass
+                except Exception as exc:
+                    LOG.debug("Failed decoding buffer\nError: %s\n", repr(exc))
+        except streamexpect.ExpectTimeout as exc:
+            LOG.debug("Failed flushing buffer\nError: %s\n", repr(exc))

    # Send command
    stream.sendall(f"{cmd}\n".encode('utf-8'))
@ -110,8 +113,8 @@ def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, fl
        while (end_time - now) >= 0:
            try:
                incoming = stream.recv(max_read_buffer)
-            except socket.timeout:
-                pass
+            except socket.timeout as exc:
+                LOG.debug("Failed reading buffer\nError: %s\n", repr(exc))
            if incoming:
                data += incoming
                if log:
@ -133,16 +136,19 @@ def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, fl
        stream.settimeout(prev_timeout)


-def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True):
+def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True, log=True):
    """
    Wait for user specified text from stream.
    """

    time.sleep(1)
-    if timeout < 60:
-        LOG.info("Expecting text within %s seconds: %s\n", timeout, text)
-    else:
-        LOG.info("Expecting text within %s minutes: %s\n", timeout / 60, text)
+
+    if log:
+        if timeout < 60:
+            LOG.info("Expecting text within %s seconds: %s\n", timeout, text)
+        else:
+            LOG.info("Expecting text within %s minutes: %s\n", timeout / 60, text)
+
    try:
        stream.expect_bytes(f"{text}".encode('utf-8'), timeout=timeout)
    except streamexpect.ExpectTimeout:
@ -153,12 +159,13 @@ def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True):
        LOG.error("Did not find expected text")
        # disconnect(stream)
        raise
-    except Exception as exception:
-        LOG.info("Connection failed with %s", exception)
+    except Exception as exc:
+        LOG.debug("Failed connection\nError: %s\n", repr(exc))
        raise

    stdout.write('\n')
-    LOG.info("Found expected text: %s", text)
+    if log:
+        LOG.debug("Found expected text: %s", text)

    time.sleep(1)
    if flush:
@ -167,21 +174,21 @@ def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True):
            if incoming:
                incoming += b'\n'
                try:
-                    LOG.info(">>> expect_bytes: Buffer has bytes!")
+                    if log:
+                        LOG.debug(">>> expect_bytes: Buffer has bytes!")
                    stdout.write(incoming.decode('utf-8'))  # streamexpect hardcodes it
-                except Exception:  # pylint: disable=W0703
-                    pass
-        except streamexpect.ExpectTimeout:
-            pass
-
+                except Exception as exc:
+                    LOG.debug("Failed decoding buffer\nError: %s\n", repr(exc))
+        except streamexpect.ExpectTimeout as exc:
+            LOG.debug("Failed flushing buffer\nError: %s\n", repr(exc))
    return 0


 # pylint: disable=inconsistent-return-statements
-def send_bytes(stream, text, fail_ok=False, expect_prompt=True,
-               prompt=None, timeout=180, send=True, flush=True):
+def send_bytes(stream, command, fail_ok=False, expect_prompt=True,
+               prompt=None, timeout=180, send=True, flush=True, log=True):
    """
-    Send user specified text to stream.
+    Send user specified command to stream.
    """

    time.sleep(1)
@ -191,19 +198,20 @@ def send_bytes(stream, text, fail_ok=False, expect_prompt=True,
            if incoming:
                incoming += b'\n'
                try:
-                    LOG.info(">>> send_bytes: Buffer has bytes!")
+                    LOG.debug(">>> send_bytes: Buffer has bytes!")
                    stdout.write(incoming.decode('utf-8'))  # streamexpect hardcodes it
-                except Exception:  # pylint: disable=W0703
-                    pass
-        except streamexpect.ExpectTimeout:
-            pass
+                except Exception as exc:
+                    LOG.debug("Failed decoding buffer\nError: %s\n", repr(exc))
+        except streamexpect.ExpectTimeout as exc:
+            LOG.debug("Failed flushing buffer\nError: %s\n", repr(exc))

-    LOG.info("Sending text: %s", text)
+    if log:
+        LOG.info("Sending command: %s", command)
    try:
        if send:
-            stream.sendall(f"{text}\n".encode('utf-8'))
+            stream.sendall(f"{command}\n".encode('utf-8'))
        else:
-            stream.sendall(f"{text}".encode('utf-8'))
+            stream.sendall(f"{command}".encode('utf-8'))
        if expect_prompt:
            time.sleep(1)
            if prompt:
@ -218,11 +226,12 @@ def send_bytes(stream, text, fail_ok=False, expect_prompt=True,
        if fail_ok:
            return -1

-        LOG.error("Failed to send text, logging out.")
+        LOG.error("Failed to send command, logging out.")
        stream.sendall("exit".encode('utf-8'))
        raise
-    except Exception as exception:
-        LOG.info("Connection failed with %s.", exception)
+    except Exception as exc:
+        LOG.error("Connection failed")
+        LOG.debug("Failed flushing buffer\nError: %s\n", repr(exc))
        raise

    return 0
--- a/virtualbox/pybox/utils/sftp.py
+++ b/virtualbox/pybox/utils/sftp.py
@ -11,8 +11,8 @@ rsync and paramiko libraries.
 import getpass
 import os
 import time
-import subprocess
 import paramiko
+from helper.install_lab import exec_cmd
 from utils.install_log import LOG


@ -47,6 +47,7 @@ def sftp_send(source, destination, client_dict):
        except Exception:  # pylint: disable=W0703
            LOG.info("******* try again")
            retry += 1
+            LOG.info("Waiting 10s")
            time.sleep(10)

    LOG.info("Sending file from %s to %s", source, destination)
@ -102,25 +103,15 @@ def send_dir(params_dict):
            keygen_arg = f"[127.0.0.1]:{remote_port}"
        else:
            keygen_arg = remote_host
-        cmd = f'ssh-keygen -f "/home/{getpass.getuser()}/.ssh/known_hosts" -R {keygen_arg}'
-        LOG.info("CMD: %s", cmd)
-        with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process:
-            for line in iter(process.stdout.readline, b''):
-                LOG.info("%s", line.decode("utf-8").strip())
-            process.wait()
+        cmd = f'ssh-keygen -f \
+                    "/home/{getpass.getuser()}/.ssh/known_hosts" -R {keygen_arg} 2>/dev/null'
+        exec_cmd(cmd)

-    LOG.info('Running rsync of dir: %s -> %s@%s:%s', source, username, remote_host, destination)
+    LOG.info('#### Running rsync of dir: %s -> %s@%s:%s', source, username, remote_host, destination)
    cmd = (f'rsync -av{follow_links} --rsh="/usr/bin/sshpass -p {password} '
           f'ssh -p {remote_port} -o StrictHostKeyChecking=no -l {username}" '
           f'{source}* {username}@{remote_host}:{destination}')
-    LOG.info("CMD: %s", cmd)
-
-    with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process:
-        for line in iter(process.stdout.readline, b''):
-            LOG.info("%s", line.decode("utf-8").strip())
-        process.wait()
-    if process.returncode:
-        raise Exception(f"Error in rsync, return code:{process.returncode}")  # pylint: disable=E0012, W0719
+    exec_cmd(cmd)


 def send_dir_fallback(source, remote_host, destination, username, password):
@ -165,4 +156,5 @@ def send_dir_fallback(source, remote_host, destination, username, password):
    sftp_client.close()
    ssh_client.close()
    if send_img:
+        LOG.info("Waiting 10s")
        time.sleep(10)
--- a/virtualbox/pybox/utils/tests/test_serial.py
+++ b/virtualbox/pybox/utils/tests/test_serial.py
@ -48,7 +48,7 @@ class ConnectTestCase(unittest.TestCase):
        # Assert
        mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
        mock_socket.connect.assert_called_once_with(('localhost', port))
-        mock_log_info.assert_called_with("Connection failed")
+        mock_log_info.assert_called_with("Failed sock connection")
        self.assertIsNone(result)


@ -116,9 +116,10 @@ class ExpectBytesTestCase(unittest.TestCase):
    Class to test expect_bytes method
    """

+    @patch("serial.LOG.debug")
    @patch("serial.LOG.info")
    @patch("serial.stdout.write")
-    def test_expect_bytes(self, mock_stdout_write, mock_log_info):
+    def test_expect_bytes(self, mock_stdout_write, mock_log_info, mock_log_debug):
        """
        Test expect_bytes method
        """
@ -140,7 +141,7 @@ class ExpectBytesTestCase(unittest.TestCase):
        stream.expect_bytes.assert_called_once_with(f"{text}".encode('utf-8'), timeout=timeout)
        mock_stdout_write.assert_any_call('\n')
        mock_log_info.assert_any_call("Expecting text within %s minutes: %s\n", timeout / 60, text)
-        mock_log_info.assert_any_call("Found expected text: %s", text)
+        mock_log_debug.assert_any_call("Found expected text: %s", text)


 class SendBytesTestCase(unittest.TestCase):
--- a/virtualbox/pybox/utils/tests/test_sftp.py
+++ b/virtualbox/pybox/utils/tests/test_sftp.py
@ -54,7 +54,7 @@ class TestSendDir(unittest.TestCase):
    """

    @patch("serial.LOG.info")
-    @patch('sftp.subprocess.Popen')
+    @patch('sftp.exec_cmd')
    @patch('sftp.getpass.getuser')
    def test_send_dir(self, mock_getuser, mock_popen, mock_log_info):
        """
--- a/virtualbox/pybox/vbox-controlgrp.sh
+++ b/virtualbox/pybox/vbox-controlgrp.sh
@ -41,7 +41,7 @@ SNAP_ACTIONS="take delete restore"

 get_vms_by_group () {
    local group=$1
-    vms=$(VBoxManage list -l vms |
+    vms=$(vboxmanage list -l vms |
        awk -v group="/$group" \
        '/^Name:/ { name = $2; } '`
        '/^Groups:/ { groups = $2; } '`
@ -62,7 +62,7 @@ if [[ "$SNAP_ACTIONS" = *"$ACTION"* ]]; then
    while read -r vm; do
        vm=(${vm})
        echo "Executing '$ACTION' on ${vm[0]}..."
-        VBoxManage snapshot ${vm[1]} "${ACTION}" "${SNAP_NAME}"
+        vboxmanage snapshot ${vm[1]} "${ACTION}" "${SNAP_NAME}"
    done <<< "$vms"
 elif [[ "$BASIC_INST_ACTIONS" = *"$ACTION"* ]]; then
    vms=$(get_vms_by_group "$GROUP")
@ -70,7 +70,7 @@ elif [[ "$BASIC_INST_ACTIONS" = *"$ACTION"* ]]; then
    while read -r vm; do
        vm=(${vm})
        echo "Executing '$ACTION' on '${vm[0]}'..."
-        VBoxManage controlvm ${vm[1]} "${ACTION}"
+        vboxmanage controlvm ${vm[1]} "${ACTION}"
    done <<< "$vms"
    wait
 elif [[ "$ACTION" = "resume" ]]; then