Add support for 'crash' reboot method

Change-Id: Iffae9115d858d8038287eaf764fe84f9bcf10ef5
This commit is contained in:
Julia Marciano 2021-01-27 23:23:41 +02:00 committed by Federico Ressi
parent d7e8a65948
commit 7875210146
5 changed files with 82 additions and 70 deletions

View File

@ -43,6 +43,7 @@ activate_server = _client.activate_server
ensure_server_status = _client.ensure_server_status
migrate_server = _client.migrate_server
confirm_resize = _client.confirm_resize
reboot_server = _client.reboot_server
NovaServer = _client.NovaServer
WaitForCloudInitTimeoutError = _cloud_init.WaitForCloudInitTimeoutError

View File

@ -238,7 +238,7 @@ class WaitForServerStatusTimeout(WaitForServerStatusError):
NOVA_SERVER_TRANSIENT_STATUS: typing.Dict[str, typing.List[str]] = {
'ACTIVE': ['BUILD', 'SHUTOFF'],
'ACTIVE': ['BUILD', 'SHUTOFF', 'REBOOT'],
'SHUTOFF': ['ACTIVE'],
'VERIFY_RESIZE': ['RESIZE'],
}
@ -324,7 +324,7 @@ def activate_server(server: ServerType,
LOG.info(f"Confirm resize of server '{server.id}' "
f"(status='{server.status}').")
client.servers.confirm_resize(server)
else:
elif server.status != 'REBOOT':
LOG.warning(f"Try activating server '{server.id}' by rebooting "
f"it (status='{server.status}').")
client.servers.reboot(server.id, reboot_type='HARD')
@ -334,6 +334,27 @@ def activate_server(server: ServerType,
sleep_time=sleep_time)
def reboot_server(server: ServerType,
client: NovaClientType = None,
timeout: tobiko.Seconds = None,
sleep_time: tobiko.Seconds = None) -> NovaServer:
client = nova_client(client)
server = get_server(server=server, client=client)
if server.status == 'REBOOT':
return server
if server.status == 'SHUTOFF':
LOG.info(f"Start server '{server.id}' (status='{server.status}').")
client.servers.start(server.id)
else:
LOG.info(f"Reboot server '{server.id}' (status='{server.status}').")
client.servers.reboot(server.id)
return wait_for_server_status(server=server.id, status='ACTIVE',
client=client, timeout=timeout,
sleep_time=sleep_time)
def ensure_server_status(server: ServerType,
status: str,
client: NovaClientType = None,

View File

@ -72,8 +72,10 @@ reboot_host = _reboot.reboot_host
RebootHostError = _reboot.RebootHostError
RebootHostOperation = _reboot.RebootHostOperation
RebootHostTimeoutError = _reboot.RebootHostTimeoutError
hard_reset_method = _reboot.hard_reset_method
soft_reset_method = _reboot.soft_reset_method
RebootHostMethod = _reboot.RebootHostMethod
crash_method = RebootHostMethod.CRASH
hard_reset_method = RebootHostMethod.HARD
soft_reset_method = RebootHostMethod.SOFT
ssh_process = _ssh.ssh_process
ssh_execute = _ssh.ssh_execute

View File

@ -13,19 +13,28 @@
# under the License.
from __future__ import absolute_import
import enum
import typing # noqa
from oslo_log import log
import tobiko
from tobiko.shell.sh import _command
from tobiko.shell.sh import _uptime
from tobiko.shell import ssh
LOG = log.getLogger(__name__)
hard_reset_method = 'echo b > /proc/sysrq-trigger'
soft_reset_method = '/sbin/reboot'
class RebootHostMethod(enum.Enum):
SOFT = '/sbin/reboot',
HARD = 'echo 1 > /proc/sys/kernel/sysrq && echo b > /proc/sysrq-trigger',
CRASH = 'echo 1 > /proc/sys/kernel/sysrq && echo c > /proc/sysrq-trigger',
def __init__(self, command: str):
self.command = command
class RebootHostError(tobiko.TobikoException):
@ -39,53 +48,37 @@ class RebootHostTimeoutError(RebootHostError):
def reboot_host(ssh_client: ssh.SSHClientFixture,
wait: bool = True,
timeout: tobiko.Seconds = None,
method: str = None,
hard: bool = False):
if method not in (None, hard_reset_method, soft_reset_method):
raise ValueError(f"Unsupported method: '{method}'")
command = method or (hard and hard_reset_method) or None
method: RebootHostMethod = RebootHostMethod.SOFT):
reboot = RebootHostOperation(ssh_client=ssh_client,
wait=wait,
timeout=timeout,
command=command)
return tobiko.setup_fixture(reboot)
method=method)
tobiko.setup_fixture(reboot)
if wait:
reboot.wait_for_operation()
return reboot
class RebootHostOperation(tobiko.Operation):
hostname = None
is_rebooted: typing.Optional[bool] = None
start_time: tobiko.Seconds = None
default_wait_timeout = 300.
default_wait_interval = 5.
default_wait_count = 60
command = soft_reset_method
@property
def ssh_client(self) -> ssh.SSHClientFixture:
if self._ssh_client is None:
raise ValueError(f"SSH client for object '{self}' is None")
return self._ssh_client
def __init__(self,
ssh_client: typing.Optional[ssh.SSHClientFixture] = None,
wait: bool = True,
ssh_client: ssh.SSHClientFixture,
timeout: tobiko.Seconds = None,
command: typing.Optional[str] = None):
method: RebootHostMethod = RebootHostMethod.SOFT):
super(RebootHostOperation, self).__init__()
self._ssh_client = ssh_client
tobiko.check_valid_type(self.ssh_client, ssh.SSHClientFixture)
self.wait = bool(wait)
tobiko.check_valid_type(ssh_client, ssh.SSHClientFixture)
tobiko.check_valid_type(method, RebootHostMethod)
self.is_rebooted = False
self.method = method
self.ssh_client = ssh_client
self.start_time: tobiko.Seconds = None
self.timeout = tobiko.to_seconds(timeout)
if command is not None:
self.command = command
def run_operation(self):
ssh_client = self.ssh_client
self.is_rebooted = None
self.is_rebooted = False
self.start_time = None
for attempt in tobiko.retry(
timeout=self.timeout,
@ -93,14 +86,13 @@ class RebootHostOperation(tobiko.Operation):
default_count=self.default_wait_count,
default_interval=self.default_wait_interval):
try:
channel = ssh_client.connect(
channel = self.ssh_client.connect(
connection_timeout=attempt.time_left,
retry_count=1)
self.hostname = self.hostname or ssh_client.hostname
LOG.info("Executing reboot command on host "
f"'{self.hostname}' (command='{self.command}')... ")
self.start_time = tobiko.time()
channel.exec_command(f"sudo /bin/sh -c '{self.command}'")
channel.exec_command(str(self.command))
except Exception as ex:
if attempt.time_left > 0.:
LOG.debug(f"Unable to reboot remote host "
@ -108,25 +100,29 @@ class RebootHostOperation(tobiko.Operation):
else:
LOG.exception(f"Unable to reboot remote host: {ex}")
raise RebootHostTimeoutError(
hostname=self.hostname or ssh_client.host,
hostname=self.hostname or self.ssh_client.host,
timeout=attempt.timeout) from ex
else:
self.is_rebooted = False
LOG.info(f"Host '{self.hostname}' is rebooting "
f"(command='{self.command}').")
break
finally:
# Ensure we close connection after rebooting command
ssh_client.close()
if self.wait:
self.wait_for_operation()
self.ssh_client.close()
def cleanup_fixture(self):
self.is_rebooted = None
self.hostname = None
self.is_rebooted = False
self.start_time = None
@property
def command(self) -> _command.ShellCommand:
return _command.shell_command(
['sudo', '/bin/sh', '-c', self.method.command])
@property
def hostname(self) -> str:
return self.ssh_client.hostname
@property
def elapsed_time(self) -> tobiko.Seconds:
if self.start_time is None:
@ -166,7 +162,6 @@ class RebootHostOperation(tobiko.Operation):
f"'{self.hostname}'", exc_info=1)
attempt.check_limits()
else:
# verify that reboot actually happened by comparing elapsed
# time with up_time
elapsed_time = self.elapsed_time
@ -184,8 +179,4 @@ class RebootHostOperation(tobiko.Operation):
attempt.check_limits()
finally:
if not self.is_rebooted:
try:
tobiko.cleanup_fixture(self.ssh_client)
except Exception:
LOG.exception("Error closing SSH connection to "
f"'{self.hostname}'")
self.ssh_client.close()

View File

@ -21,6 +21,7 @@ from oslo_log import log
import testtools
import tobiko
from tobiko.shell import ping
from tobiko.shell import sh
from tobiko.openstack import nova
from tobiko.openstack import stacks
@ -37,7 +38,7 @@ class RebootHostTest(testtools.TestCase):
stack = tobiko.required_setup_fixture(RebootHostStack)
def test_reboot_host(self, **params):
def test_reboot_host(self, nova_reboot=False, **params):
server = self.stack.ensure_server_status('ACTIVE')
self.assertEqual('ACTIVE', server.status)
@ -53,19 +54,16 @@ class RebootHostTest(testtools.TestCase):
timeout=90.)
reboot = sh.reboot_host(ssh_client=ssh_client, **params)
self.assertIs(ssh_client, reboot.ssh_client)
self.assertEqual(ssh_client.hostname, reboot.hostname)
self.assertIs(params.get('wait', True), reboot.wait)
hard = params.get('hard', False)
command = (params.get('method') or
(hard and sh.hard_reset_method) or
sh.soft_reset_method)
self.assertEqual(command, reboot.command)
method = params.get('method') or sh.soft_reset_method
self.assertIs(method, reboot.method)
if not reboot.wait:
self.assertFalse(reboot.is_rebooted)
if not reboot.is_rebooted:
self.assert_is_not_connected(ssh_client)
if nova_reboot:
ping.ping_until_unreceived(self.stack.ip_address)
nova.reboot_server(server)
reboot.wait_for_operation()
self.assertTrue(reboot.is_rebooted)
@ -80,8 +78,10 @@ class RebootHostTest(testtools.TestCase):
"uptime=%r", uptime_1)
self.assertGreater(boottime_1, boottime_0)
def test_reboot_host_with_hard(self):
self.test_reboot_host(hard=True)
def test_reboot_host_with_chash_method(self):
self.test_reboot_host(method=sh.crash_method,
wait=False,
nova_reboot=True)
def test_reboot_host_with_hard_method(self):
self.test_reboot_host(method=sh.hard_reset_method)
@ -90,14 +90,11 @@ class RebootHostTest(testtools.TestCase):
self.test_reboot_host(method=sh.soft_reset_method)
def test_reboot_host_with_invalid_method(self):
self.assertRaises(ValueError,
self.assertRaises(TypeError,
sh.reboot_host,
ssh_client=self.stack.ssh_client,
method='<invalid-method>')
def test_reboot_host_with_no_hard(self):
self.test_reboot_host(hard=False)
def test_reboot_host_with_wait(self):
self.test_reboot_host(wait=True)