test/automated-pytest-suite/utils/multi_thread.py

#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#


import threading
import time
import traceback

from consts.proj_vars import ProjVar
from utils.clients.ssh import SSHClient, ControllerClient, NATBoxClient
from utils.exceptions import ThreadingError
from utils.tis_log import LOG

TIMEOUT_ERR = "Thread did not terminate within timeout. " \
              "Thread details: {} {} {}"
EVENT_TIMEOUT = "Event did not occur within timeout."
INFINITE_WAIT_EVENT_EXPECTED = "There is only one thread (Main Thread), " \
                               "waiting is likely to wait indefinitely"


class MThread(threading.Thread):
    """
    Multi threading class. Allows multiple threads to be run simultaneously.
    e.g. nova_helper.create_flavor('threading', 'auto', vcpus=2, ram=1024) is
    equivalent to...
        thread_1 = MThread(nova_helper.create_flavor, 'threading', 'auto',
        vcpus=2, ram=1024)
        thread_1.start_thread()
        thread_1.wait_for_thread_end()

    Other commands can be run between start_thread and wait_for_thread_end
    The function's output can be retrieved from thread_1.get_output()
    name should NOT be changed
    """
    total_threads = 0
    running_threads = []

    def __init__(self, func, *args, **kwargs):
        """

        Args:
            func (runnable): name of function to run. e.g.
            nova-helper.create_flavor. NOT nova_helper.create_flavor()
            *args:
            **kwargs:
        """
        threading.Thread.__init__(self)
        MThread.total_threads += 1
        self.thread_id = MThread.total_threads
        self.func = func
        self.args = args
        self.kwargs = kwargs
        self._output = None
        self._output_returned = threading.Event()
        self.timeout = None
        self._err = None

    def get_output(self, wait=True, timeout=None):
        """
        Get return value of self.func

        Args:
            wait (bool): Whether or not to wait for the output from self.func
            timeout:

        Returns: return value from self.func

        """
        if wait:
            if timeout is None:
                timeout = self.timeout

            if timeout:
                end_time = time.time() + timeout
                while time.time() < end_time:
                    if self._err:
                        raise ThreadingError(str(self._err))
                    if self._output_returned.is_set():
                        # LOG.info("{}: {} returned: {}".format(self.name,
                        # self.func.__name__, self._output.__str__))
                        break

        return self._output

    def get_error_info(self):
        return self._err

    def start_thread(self, timeout=3600):
        """
        Starts a thread.
        Test must wait for thread to terminate or it can continue running
        during other tests.

        Args:
            timeout (int): how long to wait for the thread to finish

        Returns:

        """
        self.timeout = timeout
        self.__start_thread_base()

    def __start_thread_base(self):
        self.start()

    def run(self):
        """
        Do not run this command. Start threads from start_thread functions
        Returns:

        """
        LOG.info("Starting {}".format(self.name))
        # run the function
        try:
            MThread.running_threads.append(self)
            LOG.info("Connecting to lab fip in new thread...")
            lab = ProjVar.get_var('lab')
            lab_fip = lab['floating ip']
            con_ssh = SSHClient(lab_fip)
            con_ssh.connect(use_current=False)
            ControllerClient.set_active_controller(con_ssh)

            if ProjVar.get_var('IS_DC'):
                LOG.info("Connecting to subclouds fip in new thread...")
                ControllerClient.set_active_controller(con_ssh, 'RegionOne')
                con_ssh_dict = ControllerClient.get_active_controllers_map()
                for name in con_ssh_dict:
                    if name in lab:
                        subcloud_fip = lab[name]['floating ip']
                        subcloud_ssh = SSHClient(subcloud_fip)
                        try:
                            subcloud_ssh.connect(use_current=False)
                            ControllerClient.set_active_controller(subcloud_ssh,
                                                                   name=name)
                        except:
                            if name == ProjVar.get_var('PRIMARY_SUBCLOUD'):
                                raise
                            LOG.warning('Cannot connect to {}'.format(name))

            LOG.info("Connecting to NatBox in new thread...")
            NATBoxClient.set_natbox_client()

            LOG.info("Execute function {}({}, {})".format(self.func.__name__,
                                                          self.args,
                                                          self.kwargs))
            self._output = self.func(*self.args, **self.kwargs)
            LOG.info("{} returned: {}".format(self.func.__name__,
                                              self._output.__str__()))
            self._output_returned.set()
        except:
            err = traceback.format_exc()
            # LOG.error("Error found in thread call {}".format(err))
            self._err = err
            raise
        finally:
            LOG.info("Terminating thread: {}".format(self.thread_id))
            if ProjVar.get_var('IS_DC'):
                ssh_clients = ControllerClient.get_active_controllers(
                    current_thread_only=True)
                for con_ssh in ssh_clients:
                    con_ssh.close()
            else:
                ControllerClient.get_active_controller().close()

            natbox_ssh = NATBoxClient.get_natbox_client()
            if natbox_ssh:
                natbox_ssh.close()

            LOG.debug("{} has finished".format(self.name))
            MThread.running_threads.remove(self)

    def wait_for_thread_end(self, timeout=3600, fail_ok=False):
        """
        Waits for thread (self) to finish executing.
        All tests should wait for threads to end before proceeding to
        teardown, unless it is expected to continue,
        e.g. LOG.tc_step will not work during setup or teardown
        Raise error if thread is still running after timeout
        Args:
            timeout (int): how long to wait for the thread to finish.
            self.timeout is preferred.
            fail_ok (bool): fail_ok=False will raise error if wait times out
            or fails test if thread exited due to error

        Returns (bool): True if thread is not running, False/exception otherwise

        """
        if not self.is_alive():
            LOG.info("{} was already finished".format(self.name))
            if self._err:
                if not fail_ok:
                    raise ThreadingError(
                        "Error in thread: {}".format(self._err))
                LOG.error("Error found in thread call {}".format(self._err))
            return True, self._err

        if not timeout:
            timeout = self.timeout

        LOG.info("Wait for {} to finish".format(self.name))
        self.join(timeout)

        if not fail_ok:
            assert not self._err, "{} ran into an error: {}".format(self.name,
                                                                    self._err)

        if not self.is_alive():
            LOG.info("{} has finished".format(self.name))
        else:
            # Thread didn't finish before timeout
            LOG.error("{} did not finish within timeout".format(self.name))
            if fail_ok:
                return False, self._err
            raise ThreadingError(
                TIMEOUT_ERR.format(self.func, self.args, self.kwargs))

        return True, self._err


def get_multi_threads():
    return MThread.running_threads


def is_multi_thread_active():
    if len(get_multi_threads()) == 0:
        return False
    return True


class Events(threading.Event):
    """
    Blocks thread progress when wait_for_event is called
    (Main Thread)               (Thread-1)
    event = Events()
    event.wait_for_event()
    # waits
                                event.set()
    # continues
    """

    def __init__(self, message="Simple event"):
        threading.Event.__init__(self)
        self.message = message
        self.msg_lock = threading.Lock()

    def set(self):
        """
        Set event flag to true. Allows threads waiting for the event to continue
        """
        threading.Event.set(self)
        LOG.info("Event \"{}\" flag set to True".format(self.message))

    def clear(self):
        threading.Event.clear(self)
        LOG.info("Event \"{}\" flag set to False".format(self.message))

    def wait_for_event(self, timeout=3600, fail_ok=False):
        """
        Waits for this Events object to have its flag set to True
        Args:
            timeout:
            fail_ok:

        Returns (bool): True if event flag set to true, False/exception
        otherwise

        """
        if self.is_set():
            LOG.info("Event \"{}\" has already been set".format(self.message))
            return True
        if not timeout:
            LOG.warning(
                "No timeout was specified. This can lead to waiting "
                "indefinitely")

        if not is_multi_thread_active() and not timeout:
            LOG.error(
                "There are no other running threads that can set this event "
                "to true. "
                "This would wait indefinitely")
            raise ThreadingError(INFINITE_WAIT_EVENT_EXPECTED)

        LOG.info(
            "Waiting for event \"{}\" flag set to true".format(self.message))
        if not threading.Event.wait(self, timeout):
            if fail_ok:
                LOG.error(
                    "Timed out waiting for event \"{}\" flag set to "
                    "True".format(
                        self.message))
                return False
            raise ThreadingError(EVENT_TIMEOUT)

        if self.msg_lock.acquire(False):
            # only one message should appear once event is set
            LOG.info("Threads continuing")
            time.sleep(1)
            self.msg_lock.release()
        return True


class TiSBarrier(threading.Barrier):
    """
    Blocks threads calling wait() until specified number (parties) of threads
    are waiting or timeout expires.
    (Main Thread)               (Thread-1)              (Thread-2)
    barr = TiSBarrier(2)
                                barr.wait()
                                # Thread-1 waits
                                                        barr.wait()
                                # both continue
    """

    def __init__(self, parties, action=None, timeout=180):
        """
        Args:
            parties (int): number of threads to wait for
            action (function): additional function to call when barrier breaks
            timeout (int): max wait time
        """
        if len(get_multi_threads()) + 1 < parties:
            LOG.warning(
                "This barrier will wait for more threads than are currently "
                "running")
        threading.Barrier.__init__(self, parties, action, timeout)

        self.timeout = timeout
        LOG.info("Created a barrier waiting for {} threads".format(parties))

    def wait(self, timeout=None):
        """
        Block thread until barrier breaks
        Exception is raised if timeout expires
        Args:
            timeout (int): max wait time at barrier. Preferred over
            self.timeout, if given

        Returns (int): number from 0 to (parties - 1), randomly assigned and
        unique for each thread that waited.
                       -1 if thread did not wait

        """
        self.barrier_status()
        if self.broken:
            return -1
        if not timeout:
            if not self.timeout:
                LOG.warning("This thread does not have a timeout for wait()")
            else:
                timeout = self.timeout

        try:
            LOG.info("Start waiting at barrier")
            id_ = threading.Barrier.wait(self, timeout)
            # only one thread will report the barrier being passed
            if id_ == 0:
                self._barrier_passed()
            return id
        except:
            LOG.info("Barrier broke before {} threads were waiting".format(
                self.parties))
            raise

    def barrier_status(self):
        if not self.broken:
            LOG.info(
                '{} threads are waiting... Barrier waiting for {} '
                'threads'.format(
                    self.n_waiting, self.parties))
        else:
            LOG.info("This barrier has already been passed.")

    def _barrier_passed(self):
        LOG.info("Barrier has been passed. {} threads continuing".format(
            self.parties))


class TiSLock:
    """
    RLock class with added logging
    lock can be acquired using 'lock.acquire()' or 'with lock as have_lock'
    lock released using 'lock.release()' or exiting with scope
    Do Not call __enter__ or __exit__ explicitly

    (Main Thread)               (Thread-1)
    lock = TiSLock()
    lock.acquire()
    # edit test.txt
                                lock.acquire()
                                # waits
    lock.release()
                                # continues
    """

    def __init__(self, blocking=True, timeout=180):
        """
        Create a locking object
        Args:
            blocking (bool): wait for lock to be released if it is locked
            during first acquire attempt
            timeout (int): how long to wait for lock to be released. Should
            set to -1 if blocking is false
                           -1 with blocking True will have no timeout
        """
        self._lock = threading.RLock()
        self.blocking = blocking
        self.timeout = timeout if blocking else -1

    def acquire(self, **kwargs):
        if 'blocking' in kwargs:
            self.blocking = kwargs['blocking']
        if 'timeout' in kwargs:
            self.timeout = kwargs['timeout']
        return self.__enter__()

    def release(self):
        return self.__exit__()

    def __enter__(self):
        """
        Make sure that True is returned.
        Threads will continue if timeout is reached even though lock was not
        acquired

        Returns (bool): True if lock was acquired, False otherwise

        """
        blocking_msg = "" if self.blocking else "not "
        timeout_msg = self.timeout if self.timeout >= 0 else "None"
        msg = "Attempting to acquire lock, {}blocking, timeout - {}".format(
            blocking_msg, timeout_msg)
        LOG.debug(msg)
        got_lock = False
        try:
            got_lock = self._lock.acquire(self.blocking, self.timeout)
        finally:
            if got_lock:
                LOG.debug("Acquired lock")
            else:
                LOG.debug("Could not acquire lock")
            return got_lock

    def __exit__(self, *args):
        LOG.debug("Releasing lock")
        try:
            return self._lock.release()
        except RuntimeError:
            LOG.error("Lock did not release, lock was unlocked already")
            raise
        except:
            LOG.error("An unexpected error was caught when unlocking lock")
            raise