tobiko/tobiko/shell/sh/_reboot.py

# Copyright 2020 Red Hat
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
from __future__ import absolute_import

import enum
import typing  # noqa

from oslo_log import log

import tobiko
from tobiko.shell.sh import _command
from tobiko.shell.sh import _uptime
from tobiko.shell import ssh


LOG = log.getLogger(__name__)


class RebootHostMethod(enum.Enum):

    SOFT = '/sbin/reboot',
    HARD = 'echo 1 > /proc/sys/kernel/sysrq && echo b > /proc/sysrq-trigger',
    CRASH = 'echo 1 > /proc/sys/kernel/sysrq && echo c > /proc/sysrq-trigger',

    def __init__(self, command: str):
        self.command = command


class RebootHostError(tobiko.TobikoException):
    message = "host {hostname!r} not rebooted: {cause}"


class RebootHostTimeoutError(RebootHostError):
    message = "host {hostname!r} not rebooted after {timeout!s} seconds"


def reboot_host(ssh_client: ssh.SSHClientFixture,
                wait: bool = True,
                timeout: tobiko.Seconds = None,
                method: RebootHostMethod = RebootHostMethod.SOFT):
    reboot = RebootHostOperation(ssh_client=ssh_client,
                                 timeout=timeout,
                                 method=method)
    tobiko.setup_fixture(reboot)
    if wait:
        reboot.wait_for_operation()
    return reboot


class RebootHostOperation(tobiko.Operation):

    default_wait_timeout = 300.
    default_wait_interval = 5.
    default_wait_count = 60

    def __init__(self,
                 ssh_client: ssh.SSHClientFixture,
                 timeout: tobiko.Seconds = None,
                 method: RebootHostMethod = RebootHostMethod.SOFT):
        super(RebootHostOperation, self).__init__()
        tobiko.check_valid_type(ssh_client, ssh.SSHClientFixture)
        tobiko.check_valid_type(method, RebootHostMethod)
        self.is_rebooted = False
        self.method = method
        self.ssh_client = ssh_client
        self.start_time: tobiko.Seconds = None
        self.timeout = tobiko.to_seconds(timeout)

    def run_operation(self):
        self.is_rebooted = False
        self.start_time = None
        for attempt in tobiko.retry(
                timeout=self.timeout,
                default_timeout=self.default_wait_timeout,
                default_count=self.default_wait_count,
                default_interval=self.default_wait_interval):
            try:
                channel = self.ssh_client.connect(
                    connection_timeout=attempt.time_left,
                    retry_count=1)
                LOG.info("Executing reboot command on host "
                         f"'{self.hostname}' (command='{self.command}')... ")
                self.start_time = tobiko.time()
                channel.exec_command(str(self.command))
            except Exception as ex:
                if attempt.time_left > 0.:
                    LOG.debug(f"Unable to reboot remote host "
                              f"(time_left={attempt.time_left}): {ex}")
                else:
                    LOG.exception(f"Unable to reboot remote host: {ex}")
                    raise RebootHostTimeoutError(
                        hostname=self.hostname or self.ssh_client.host,
                        timeout=attempt.timeout) from ex
            else:
                LOG.info(f"Host '{self.hostname}' is rebooting "
                         f"(command='{self.command}').")
                break
            finally:
                # Ensure we close connection after rebooting command
                self.ssh_client.close()

    def cleanup_fixture(self):
        self.is_rebooted = False
        self.start_time = None

    @property
    def command(self) -> _command.ShellCommand:
        return _command.shell_command(
            ['sudo', '/bin/sh', '-c', self.method.command])

    @property
    def hostname(self) -> str:
        return self.ssh_client.hostname

    @property
    def elapsed_time(self) -> tobiko.Seconds:
        if self.start_time is None:
            return None
        else:
            return tobiko.time() - self.start_time

    @property
    def time_left(self) -> tobiko.Seconds:
        if self.timeout is None or self.elapsed_time is None:
            return None
        else:
            return self.timeout - self.elapsed_time

    def wait_for_operation(self, timeout: tobiko.Seconds = None):
        if self.is_rebooted:
            return
        try:
            for attempt in tobiko.retry(
                    timeout=tobiko.min_seconds(timeout, self.time_left),
                    default_timeout=self.default_wait_timeout,
                    default_count=self.default_wait_count,
                    default_interval=self.default_wait_interval):
                # ensure SSH connection is closed before retrying connecting
                tobiko.cleanup_fixture(self.ssh_client)
                assert self.ssh_client.client is None
                LOG.debug(f"Getting uptime after reboot '{self.hostname}' "
                          "after reboot... ")
                try:
                    up_time = _uptime.get_uptime(ssh_client=self.ssh_client,
                                                 timeout=30.)

                except Exception:
                    # if disconnected while getting up time we assume the VM is
                    # just rebooting. These are good news!
                    LOG.debug("Unable to get uptime from host "
                              f"'{self.hostname}'", exc_info=1)
                    attempt.check_limits()
                else:
                    # verify that reboot actually happened by comparing elapsed
                    # time with up_time
                    elapsed_time = self.elapsed_time
                    if up_time < elapsed_time:
                        assert self.ssh_client.client is not None
                        self.is_rebooted = True
                        LOG.debug(f"Host '{self.hostname}' restarted "
                                  f"{elapsed_time} seconds after "
                                  f"reboot operation (up_time={up_time})")
                        break
                    else:
                        LOG.debug(f"Host '{self.hostname}' still not "
                                  f"restarted {elapsed_time} seconds after "
                                  f"reboot operation (up_time={up_time!r})")
                        attempt.check_limits()
        finally:
            if not self.is_rebooted:
                self.ssh_client.close()