x/tobiko
Code Issues Proposed changes

Fix reboot when target host si not reachable

Browse Source 
Change-Id: I2467feac8e2a2bd6406b476c50f7fe64e9f27ca1
This commit is contained in:
Federico Ressi 2021-02-01 10:49:56 +01:00
parent 1ade9dd7ab
commit cba8d571bf
3 changed files with 83 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
tobiko/shell/sh/_reboot.py
View File

@ -18,17 +18,14 @@ import typing # noqa
from oslo_log import log
import tobiko
from tobiko.shell.sh import _exception
from tobiko.shell.sh import _uptime
from tobiko.shell import ssh
LOG = log.getLogger(__name__)
hard_reset_method = 'sudo chmod o+w /proc/sysrq-trigger;' \
'sudo echo b > /proc/sysrq-trigger'
soft_reset_method = 'sudo /sbin/reboot'
hard_reset_method = 'echo b > /proc/sysrq-trigger'
soft_reset_method = '/sbin/reboot'
class RebootHostError(tobiko.TobikoException):
@ -39,10 +36,19 @@ class RebootHostTimeoutError(RebootHostError):
message = "host {hostname!r} not rebooted after {timeout!s} seconds"
def reboot_host(ssh_client: ssh.SSHClientFixture, wait: bool = True,
timeout: tobiko.Seconds = None, method=soft_reset_method):
reboot = RebootHostOperation(ssh_client=ssh_client, wait=wait,
timeout=timeout, method=method)
def reboot_host(ssh_client: ssh.SSHClientFixture,
wait: bool = True,
timeout: tobiko.Seconds = None,
method: str = None,
hard: bool = False):
if method not in (None, hard_reset_method, soft_reset_method):
raise ValueError(f"Unsupported method: '{method}'")
command = method or (hard and hard_reset_method) or None
reboot = RebootHostOperation(ssh_client=ssh_client,
wait=wait,
timeout=timeout,
command=command)
return tobiko.setup_fixture(reboot)
@ -56,6 +62,8 @@ class RebootHostOperation(tobiko.Operation):
default_wait_interval = 5.
default_wait_count = 60
command = soft_reset_method
@property
def ssh_client(self) -> ssh.SSHClientFixture:
if self._ssh_client is None:
@ -64,28 +72,53 @@ class RebootHostOperation(tobiko.Operation):
def __init__(self,
ssh_client: typing.Optional[ssh.SSHClientFixture] = None,
wait=True,
wait: bool = True,
timeout: tobiko.Seconds = None,
method=soft_reset_method):
command: typing.Optional[str] = None):
super(RebootHostOperation, self).__init__()
self._ssh_client = ssh_client
tobiko.check_valid_type(self.ssh_client, ssh.SSHClientFixture)
self.wait = bool(wait)
self.timeout = tobiko.to_seconds(timeout)
self.method = method
if command is not None:
self.command = command
def run_operation(self):
ssh_client = self.ssh_client
with ssh_client:
self.hostname = ssh_client.hostname
LOG.debug(f"Rebooting host '{self.hostname}'... ")
self.is_rebooted = False
self.start_time = tobiko.time()
self.is_rebooted = None
self.start_time = None
for attempt in tobiko.retry(
timeout=self.timeout,
default_timeout=self.default_wait_timeout,
default_count=self.default_wait_count,
default_interval=self.default_wait_interval):
try:
ssh_client.connect(connection_timeout=self.timeout).\
exec_command(self.method)
except _exception.ShellTimeoutExpired as ex:
LOG.debug(f"Reboot command timeout expired: {ex}")
channel = ssh_client.connect(
connection_timeout=attempt.time_left,
retry_count=1)
self.hostname = self.hostname or ssh_client.hostname
LOG.info("Executing reboot command on host "
f"'{self.hostname}' (command='{self.command}')... ")
self.start_time = tobiko.time()
channel.exec_command(f"sudo /bin/sh -c '{self.command}'")
except Exception as ex:
if attempt.time_left > 0.:
LOG.debug(f"Unable to reboot remote host "
f"(time_left={attempt.time_left}): {ex}")
else:
LOG.exception(f"Unable to reboot remote host: {ex}")
raise RebootHostTimeoutError(
hostname=self.hostname or ssh_client.host,
timeout=attempt.timeout) from ex
else:
self.is_rebooted = False
LOG.info(f"Host '{self.hostname}' is rebooting "
f"(command='{self.command}').")
break
finally:
# Ensure we close connection after rebooting command
ssh_client.close()
if self.wait:
self.wait_for_operation()

34
tobiko/tests/functional/shell/test_reboot.py
View File

@ -18,7 +18,6 @@ from __future__ import absolute_import
import time
from oslo_log import log
import paramiko
import testtools
import tobiko
@ -31,7 +30,7 @@ LOG = log.getLogger(__name__)
class RebootHostStack(stacks.CirrosServerStackFixture):
"Server to be rebooted"
"""Server to be rebooted"""
class RebootHostTest(testtools.TestCase):
@ -58,6 +57,11 @@ class RebootHostTest(testtools.TestCase):
self.assertIs(ssh_client, reboot.ssh_client)
self.assertEqual(ssh_client.hostname, reboot.hostname)
self.assertIs(params.get('wait', True), reboot.wait)
hard = params.get('hard', False)
command = (params.get('method') or
(hard and sh.hard_reset_method) or
sh.soft_reset_method)
self.assertEqual(command, reboot.command)
if not reboot.wait:
self.assertFalse(reboot.is_rebooted)
@ -76,6 +80,24 @@ class RebootHostTest(testtools.TestCase):
"uptime=%r", uptime_1)
self.assertGreater(boottime_1, boottime_0)
def test_reboot_host_with_hard(self):
self.test_reboot_host(hard=True)
def test_reboot_host_with_hard_method(self):
self.test_reboot_host(method=sh.hard_reset_method)
def test_reboot_host_with_soft_method(self):
self.test_reboot_host(method=sh.soft_reset_method)
def test_reboot_host_with_invalid_method(self):
self.assertRaises(ValueError,
sh.reboot_host,
ssh_client=self.stack.ssh_client,
method='<invalid-method>')
def test_reboot_host_with_no_hard(self):
self.test_reboot_host(hard=False)
def test_reboot_host_with_wait(self):
self.test_reboot_host(wait=True)
@ -88,12 +110,12 @@ class RebootHostTest(testtools.TestCase):
ssh_client = self.stack.ssh_client
self.assert_is_not_connected(ssh_client)
errors = (paramiko.ssh_exception.NoValidConnectionsError,
paramiko.SSHException)
self.assertRaises(errors, sh.reboot_host, ssh_client=ssh_client,
self.assertRaises(sh.RebootHostTimeoutError,
sh.reboot_host,
ssh_client=ssh_client,
timeout=5.0)
self.assert_is_not_connected(ssh_client)
server = nova.wait_for_server_status(self.stack.server_id, 'SHUTOFF')
server = nova.get_server(self.stack.server_id)
self.assertEqual('SHUTOFF', server.status)
def assert_is_connected(self, ssh_client):

2
tox.ini
View File

@ -141,7 +141,7 @@ passenv =
*_proxy
setenv =
{[testenv]setenv}
PYTEST_TIMEOUT = 1200
PYTEST_TIMEOUT = 1800
[testenv:venv]