From 787521014629c53966cf4e5f90a2661e0b736207 Mon Sep 17 00:00:00 2001 From: Julia Marciano Date: Wed, 27 Jan 2021 23:23:41 +0200 Subject: [PATCH] Add support for 'crash' reboot method Change-Id: Iffae9115d858d8038287eaf764fe84f9bcf10ef5 --- tobiko/openstack/nova/__init__.py | 1 + tobiko/openstack/nova/_client.py | 25 +++++- tobiko/shell/sh/__init__.py | 6 +- tobiko/shell/sh/_reboot.py | 91 +++++++++----------- tobiko/tests/functional/shell/test_reboot.py | 29 +++---- 5 files changed, 82 insertions(+), 70 deletions(-) diff --git a/tobiko/openstack/nova/__init__.py b/tobiko/openstack/nova/__init__.py index 8b247eb7c..fc0ffc721 100644 --- a/tobiko/openstack/nova/__init__.py +++ b/tobiko/openstack/nova/__init__.py @@ -43,6 +43,7 @@ activate_server = _client.activate_server ensure_server_status = _client.ensure_server_status migrate_server = _client.migrate_server confirm_resize = _client.confirm_resize +reboot_server = _client.reboot_server NovaServer = _client.NovaServer WaitForCloudInitTimeoutError = _cloud_init.WaitForCloudInitTimeoutError diff --git a/tobiko/openstack/nova/_client.py b/tobiko/openstack/nova/_client.py index 6ae967706..060972a5d 100644 --- a/tobiko/openstack/nova/_client.py +++ b/tobiko/openstack/nova/_client.py @@ -238,7 +238,7 @@ class WaitForServerStatusTimeout(WaitForServerStatusError): NOVA_SERVER_TRANSIENT_STATUS: typing.Dict[str, typing.List[str]] = { - 'ACTIVE': ['BUILD', 'SHUTOFF'], + 'ACTIVE': ['BUILD', 'SHUTOFF', 'REBOOT'], 'SHUTOFF': ['ACTIVE'], 'VERIFY_RESIZE': ['RESIZE'], } @@ -324,7 +324,7 @@ def activate_server(server: ServerType, LOG.info(f"Confirm resize of server '{server.id}' " f"(status='{server.status}').") client.servers.confirm_resize(server) - else: + elif server.status != 'REBOOT': LOG.warning(f"Try activating server '{server.id}' by rebooting " f"it (status='{server.status}').") client.servers.reboot(server.id, reboot_type='HARD') @@ -334,6 +334,27 @@ def activate_server(server: ServerType, sleep_time=sleep_time) +def reboot_server(server: ServerType, + client: NovaClientType = None, + timeout: tobiko.Seconds = None, + sleep_time: tobiko.Seconds = None) -> NovaServer: + client = nova_client(client) + server = get_server(server=server, client=client) + if server.status == 'REBOOT': + return server + + if server.status == 'SHUTOFF': + LOG.info(f"Start server '{server.id}' (status='{server.status}').") + client.servers.start(server.id) + else: + LOG.info(f"Reboot server '{server.id}' (status='{server.status}').") + client.servers.reboot(server.id) + + return wait_for_server_status(server=server.id, status='ACTIVE', + client=client, timeout=timeout, + sleep_time=sleep_time) + + def ensure_server_status(server: ServerType, status: str, client: NovaClientType = None, diff --git a/tobiko/shell/sh/__init__.py b/tobiko/shell/sh/__init__.py index f12788e10..fc6847e43 100644 --- a/tobiko/shell/sh/__init__.py +++ b/tobiko/shell/sh/__init__.py @@ -72,8 +72,10 @@ reboot_host = _reboot.reboot_host RebootHostError = _reboot.RebootHostError RebootHostOperation = _reboot.RebootHostOperation RebootHostTimeoutError = _reboot.RebootHostTimeoutError -hard_reset_method = _reboot.hard_reset_method -soft_reset_method = _reboot.soft_reset_method +RebootHostMethod = _reboot.RebootHostMethod +crash_method = RebootHostMethod.CRASH +hard_reset_method = RebootHostMethod.HARD +soft_reset_method = RebootHostMethod.SOFT ssh_process = _ssh.ssh_process ssh_execute = _ssh.ssh_execute diff --git a/tobiko/shell/sh/_reboot.py b/tobiko/shell/sh/_reboot.py index ff989755a..0998c9c9e 100644 --- a/tobiko/shell/sh/_reboot.py +++ b/tobiko/shell/sh/_reboot.py @@ -13,19 +13,28 @@ # under the License. from __future__ import absolute_import +import enum import typing # noqa from oslo_log import log import tobiko +from tobiko.shell.sh import _command from tobiko.shell.sh import _uptime from tobiko.shell import ssh LOG = log.getLogger(__name__) -hard_reset_method = 'echo b > /proc/sysrq-trigger' -soft_reset_method = '/sbin/reboot' + +class RebootHostMethod(enum.Enum): + + SOFT = '/sbin/reboot', + HARD = 'echo 1 > /proc/sys/kernel/sysrq && echo b > /proc/sysrq-trigger', + CRASH = 'echo 1 > /proc/sys/kernel/sysrq && echo c > /proc/sysrq-trigger', + + def __init__(self, command: str): + self.command = command class RebootHostError(tobiko.TobikoException): @@ -39,53 +48,37 @@ class RebootHostTimeoutError(RebootHostError): def reboot_host(ssh_client: ssh.SSHClientFixture, wait: bool = True, timeout: tobiko.Seconds = None, - method: str = None, - hard: bool = False): - if method not in (None, hard_reset_method, soft_reset_method): - raise ValueError(f"Unsupported method: '{method}'") - - command = method or (hard and hard_reset_method) or None + method: RebootHostMethod = RebootHostMethod.SOFT): reboot = RebootHostOperation(ssh_client=ssh_client, - wait=wait, timeout=timeout, - command=command) - return tobiko.setup_fixture(reboot) + method=method) + tobiko.setup_fixture(reboot) + if wait: + reboot.wait_for_operation() + return reboot class RebootHostOperation(tobiko.Operation): - hostname = None - is_rebooted: typing.Optional[bool] = None - start_time: tobiko.Seconds = None - default_wait_timeout = 300. default_wait_interval = 5. default_wait_count = 60 - command = soft_reset_method - - @property - def ssh_client(self) -> ssh.SSHClientFixture: - if self._ssh_client is None: - raise ValueError(f"SSH client for object '{self}' is None") - return self._ssh_client - def __init__(self, - ssh_client: typing.Optional[ssh.SSHClientFixture] = None, - wait: bool = True, + ssh_client: ssh.SSHClientFixture, timeout: tobiko.Seconds = None, - command: typing.Optional[str] = None): + method: RebootHostMethod = RebootHostMethod.SOFT): super(RebootHostOperation, self).__init__() - self._ssh_client = ssh_client - tobiko.check_valid_type(self.ssh_client, ssh.SSHClientFixture) - self.wait = bool(wait) + tobiko.check_valid_type(ssh_client, ssh.SSHClientFixture) + tobiko.check_valid_type(method, RebootHostMethod) + self.is_rebooted = False + self.method = method + self.ssh_client = ssh_client + self.start_time: tobiko.Seconds = None self.timeout = tobiko.to_seconds(timeout) - if command is not None: - self.command = command def run_operation(self): - ssh_client = self.ssh_client - self.is_rebooted = None + self.is_rebooted = False self.start_time = None for attempt in tobiko.retry( timeout=self.timeout, @@ -93,14 +86,13 @@ class RebootHostOperation(tobiko.Operation): default_count=self.default_wait_count, default_interval=self.default_wait_interval): try: - channel = ssh_client.connect( + channel = self.ssh_client.connect( connection_timeout=attempt.time_left, retry_count=1) - self.hostname = self.hostname or ssh_client.hostname LOG.info("Executing reboot command on host " f"'{self.hostname}' (command='{self.command}')... ") self.start_time = tobiko.time() - channel.exec_command(f"sudo /bin/sh -c '{self.command}'") + channel.exec_command(str(self.command)) except Exception as ex: if attempt.time_left > 0.: LOG.debug(f"Unable to reboot remote host " @@ -108,25 +100,29 @@ class RebootHostOperation(tobiko.Operation): else: LOG.exception(f"Unable to reboot remote host: {ex}") raise RebootHostTimeoutError( - hostname=self.hostname or ssh_client.host, + hostname=self.hostname or self.ssh_client.host, timeout=attempt.timeout) from ex else: - self.is_rebooted = False LOG.info(f"Host '{self.hostname}' is rebooting " f"(command='{self.command}').") break finally: # Ensure we close connection after rebooting command - ssh_client.close() - - if self.wait: - self.wait_for_operation() + self.ssh_client.close() def cleanup_fixture(self): - self.is_rebooted = None - self.hostname = None + self.is_rebooted = False self.start_time = None + @property + def command(self) -> _command.ShellCommand: + return _command.shell_command( + ['sudo', '/bin/sh', '-c', self.method.command]) + + @property + def hostname(self) -> str: + return self.ssh_client.hostname + @property def elapsed_time(self) -> tobiko.Seconds: if self.start_time is None: @@ -166,7 +162,6 @@ class RebootHostOperation(tobiko.Operation): f"'{self.hostname}'", exc_info=1) attempt.check_limits() else: - # verify that reboot actually happened by comparing elapsed # time with up_time elapsed_time = self.elapsed_time @@ -184,8 +179,4 @@ class RebootHostOperation(tobiko.Operation): attempt.check_limits() finally: if not self.is_rebooted: - try: - tobiko.cleanup_fixture(self.ssh_client) - except Exception: - LOG.exception("Error closing SSH connection to " - f"'{self.hostname}'") + self.ssh_client.close() diff --git a/tobiko/tests/functional/shell/test_reboot.py b/tobiko/tests/functional/shell/test_reboot.py index 7c66eb677..0e60c39ac 100644 --- a/tobiko/tests/functional/shell/test_reboot.py +++ b/tobiko/tests/functional/shell/test_reboot.py @@ -21,6 +21,7 @@ from oslo_log import log import testtools import tobiko +from tobiko.shell import ping from tobiko.shell import sh from tobiko.openstack import nova from tobiko.openstack import stacks @@ -37,7 +38,7 @@ class RebootHostTest(testtools.TestCase): stack = tobiko.required_setup_fixture(RebootHostStack) - def test_reboot_host(self, **params): + def test_reboot_host(self, nova_reboot=False, **params): server = self.stack.ensure_server_status('ACTIVE') self.assertEqual('ACTIVE', server.status) @@ -53,19 +54,16 @@ class RebootHostTest(testtools.TestCase): timeout=90.) reboot = sh.reboot_host(ssh_client=ssh_client, **params) - self.assertIs(ssh_client, reboot.ssh_client) self.assertEqual(ssh_client.hostname, reboot.hostname) - self.assertIs(params.get('wait', True), reboot.wait) - hard = params.get('hard', False) - command = (params.get('method') or - (hard and sh.hard_reset_method) or - sh.soft_reset_method) - self.assertEqual(command, reboot.command) + method = params.get('method') or sh.soft_reset_method + self.assertIs(method, reboot.method) - if not reboot.wait: - self.assertFalse(reboot.is_rebooted) + if not reboot.is_rebooted: self.assert_is_not_connected(ssh_client) + if nova_reboot: + ping.ping_until_unreceived(self.stack.ip_address) + nova.reboot_server(server) reboot.wait_for_operation() self.assertTrue(reboot.is_rebooted) @@ -80,8 +78,10 @@ class RebootHostTest(testtools.TestCase): "uptime=%r", uptime_1) self.assertGreater(boottime_1, boottime_0) - def test_reboot_host_with_hard(self): - self.test_reboot_host(hard=True) + def test_reboot_host_with_chash_method(self): + self.test_reboot_host(method=sh.crash_method, + wait=False, + nova_reboot=True) def test_reboot_host_with_hard_method(self): self.test_reboot_host(method=sh.hard_reset_method) @@ -90,14 +90,11 @@ class RebootHostTest(testtools.TestCase): self.test_reboot_host(method=sh.soft_reset_method) def test_reboot_host_with_invalid_method(self): - self.assertRaises(ValueError, + self.assertRaises(TypeError, sh.reboot_host, ssh_client=self.stack.ssh_client, method='') - def test_reboot_host_with_no_hard(self): - self.test_reboot_host(hard=False) - def test_reboot_host_with_wait(self): self.test_reboot_host(wait=True)