Add support for 'crash' reboot method
Change-Id: Iffae9115d858d8038287eaf764fe84f9bcf10ef5
This commit is contained in:
parent
d7e8a65948
commit
7875210146
tobiko
openstack/nova
shell/sh
tests/functional/shell
@ -43,6 +43,7 @@ activate_server = _client.activate_server
|
||||
ensure_server_status = _client.ensure_server_status
|
||||
migrate_server = _client.migrate_server
|
||||
confirm_resize = _client.confirm_resize
|
||||
reboot_server = _client.reboot_server
|
||||
NovaServer = _client.NovaServer
|
||||
|
||||
WaitForCloudInitTimeoutError = _cloud_init.WaitForCloudInitTimeoutError
|
||||
|
@ -238,7 +238,7 @@ class WaitForServerStatusTimeout(WaitForServerStatusError):
|
||||
|
||||
|
||||
NOVA_SERVER_TRANSIENT_STATUS: typing.Dict[str, typing.List[str]] = {
|
||||
'ACTIVE': ['BUILD', 'SHUTOFF'],
|
||||
'ACTIVE': ['BUILD', 'SHUTOFF', 'REBOOT'],
|
||||
'SHUTOFF': ['ACTIVE'],
|
||||
'VERIFY_RESIZE': ['RESIZE'],
|
||||
}
|
||||
@ -324,7 +324,7 @@ def activate_server(server: ServerType,
|
||||
LOG.info(f"Confirm resize of server '{server.id}' "
|
||||
f"(status='{server.status}').")
|
||||
client.servers.confirm_resize(server)
|
||||
else:
|
||||
elif server.status != 'REBOOT':
|
||||
LOG.warning(f"Try activating server '{server.id}' by rebooting "
|
||||
f"it (status='{server.status}').")
|
||||
client.servers.reboot(server.id, reboot_type='HARD')
|
||||
@ -334,6 +334,27 @@ def activate_server(server: ServerType,
|
||||
sleep_time=sleep_time)
|
||||
|
||||
|
||||
def reboot_server(server: ServerType,
|
||||
client: NovaClientType = None,
|
||||
timeout: tobiko.Seconds = None,
|
||||
sleep_time: tobiko.Seconds = None) -> NovaServer:
|
||||
client = nova_client(client)
|
||||
server = get_server(server=server, client=client)
|
||||
if server.status == 'REBOOT':
|
||||
return server
|
||||
|
||||
if server.status == 'SHUTOFF':
|
||||
LOG.info(f"Start server '{server.id}' (status='{server.status}').")
|
||||
client.servers.start(server.id)
|
||||
else:
|
||||
LOG.info(f"Reboot server '{server.id}' (status='{server.status}').")
|
||||
client.servers.reboot(server.id)
|
||||
|
||||
return wait_for_server_status(server=server.id, status='ACTIVE',
|
||||
client=client, timeout=timeout,
|
||||
sleep_time=sleep_time)
|
||||
|
||||
|
||||
def ensure_server_status(server: ServerType,
|
||||
status: str,
|
||||
client: NovaClientType = None,
|
||||
|
@ -72,8 +72,10 @@ reboot_host = _reboot.reboot_host
|
||||
RebootHostError = _reboot.RebootHostError
|
||||
RebootHostOperation = _reboot.RebootHostOperation
|
||||
RebootHostTimeoutError = _reboot.RebootHostTimeoutError
|
||||
hard_reset_method = _reboot.hard_reset_method
|
||||
soft_reset_method = _reboot.soft_reset_method
|
||||
RebootHostMethod = _reboot.RebootHostMethod
|
||||
crash_method = RebootHostMethod.CRASH
|
||||
hard_reset_method = RebootHostMethod.HARD
|
||||
soft_reset_method = RebootHostMethod.SOFT
|
||||
|
||||
ssh_process = _ssh.ssh_process
|
||||
ssh_execute = _ssh.ssh_execute
|
||||
|
@ -13,19 +13,28 @@
|
||||
# under the License.
|
||||
from __future__ import absolute_import
|
||||
|
||||
import enum
|
||||
import typing # noqa
|
||||
|
||||
from oslo_log import log
|
||||
|
||||
import tobiko
|
||||
from tobiko.shell.sh import _command
|
||||
from tobiko.shell.sh import _uptime
|
||||
from tobiko.shell import ssh
|
||||
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
hard_reset_method = 'echo b > /proc/sysrq-trigger'
|
||||
soft_reset_method = '/sbin/reboot'
|
||||
|
||||
class RebootHostMethod(enum.Enum):
|
||||
|
||||
SOFT = '/sbin/reboot',
|
||||
HARD = 'echo 1 > /proc/sys/kernel/sysrq && echo b > /proc/sysrq-trigger',
|
||||
CRASH = 'echo 1 > /proc/sys/kernel/sysrq && echo c > /proc/sysrq-trigger',
|
||||
|
||||
def __init__(self, command: str):
|
||||
self.command = command
|
||||
|
||||
|
||||
class RebootHostError(tobiko.TobikoException):
|
||||
@ -39,53 +48,37 @@ class RebootHostTimeoutError(RebootHostError):
|
||||
def reboot_host(ssh_client: ssh.SSHClientFixture,
|
||||
wait: bool = True,
|
||||
timeout: tobiko.Seconds = None,
|
||||
method: str = None,
|
||||
hard: bool = False):
|
||||
if method not in (None, hard_reset_method, soft_reset_method):
|
||||
raise ValueError(f"Unsupported method: '{method}'")
|
||||
|
||||
command = method or (hard and hard_reset_method) or None
|
||||
method: RebootHostMethod = RebootHostMethod.SOFT):
|
||||
reboot = RebootHostOperation(ssh_client=ssh_client,
|
||||
wait=wait,
|
||||
timeout=timeout,
|
||||
command=command)
|
||||
return tobiko.setup_fixture(reboot)
|
||||
method=method)
|
||||
tobiko.setup_fixture(reboot)
|
||||
if wait:
|
||||
reboot.wait_for_operation()
|
||||
return reboot
|
||||
|
||||
|
||||
class RebootHostOperation(tobiko.Operation):
|
||||
|
||||
hostname = None
|
||||
is_rebooted: typing.Optional[bool] = None
|
||||
start_time: tobiko.Seconds = None
|
||||
|
||||
default_wait_timeout = 300.
|
||||
default_wait_interval = 5.
|
||||
default_wait_count = 60
|
||||
|
||||
command = soft_reset_method
|
||||
|
||||
@property
|
||||
def ssh_client(self) -> ssh.SSHClientFixture:
|
||||
if self._ssh_client is None:
|
||||
raise ValueError(f"SSH client for object '{self}' is None")
|
||||
return self._ssh_client
|
||||
|
||||
def __init__(self,
|
||||
ssh_client: typing.Optional[ssh.SSHClientFixture] = None,
|
||||
wait: bool = True,
|
||||
ssh_client: ssh.SSHClientFixture,
|
||||
timeout: tobiko.Seconds = None,
|
||||
command: typing.Optional[str] = None):
|
||||
method: RebootHostMethod = RebootHostMethod.SOFT):
|
||||
super(RebootHostOperation, self).__init__()
|
||||
self._ssh_client = ssh_client
|
||||
tobiko.check_valid_type(self.ssh_client, ssh.SSHClientFixture)
|
||||
self.wait = bool(wait)
|
||||
tobiko.check_valid_type(ssh_client, ssh.SSHClientFixture)
|
||||
tobiko.check_valid_type(method, RebootHostMethod)
|
||||
self.is_rebooted = False
|
||||
self.method = method
|
||||
self.ssh_client = ssh_client
|
||||
self.start_time: tobiko.Seconds = None
|
||||
self.timeout = tobiko.to_seconds(timeout)
|
||||
if command is not None:
|
||||
self.command = command
|
||||
|
||||
def run_operation(self):
|
||||
ssh_client = self.ssh_client
|
||||
self.is_rebooted = None
|
||||
self.is_rebooted = False
|
||||
self.start_time = None
|
||||
for attempt in tobiko.retry(
|
||||
timeout=self.timeout,
|
||||
@ -93,14 +86,13 @@ class RebootHostOperation(tobiko.Operation):
|
||||
default_count=self.default_wait_count,
|
||||
default_interval=self.default_wait_interval):
|
||||
try:
|
||||
channel = ssh_client.connect(
|
||||
channel = self.ssh_client.connect(
|
||||
connection_timeout=attempt.time_left,
|
||||
retry_count=1)
|
||||
self.hostname = self.hostname or ssh_client.hostname
|
||||
LOG.info("Executing reboot command on host "
|
||||
f"'{self.hostname}' (command='{self.command}')... ")
|
||||
self.start_time = tobiko.time()
|
||||
channel.exec_command(f"sudo /bin/sh -c '{self.command}'")
|
||||
channel.exec_command(str(self.command))
|
||||
except Exception as ex:
|
||||
if attempt.time_left > 0.:
|
||||
LOG.debug(f"Unable to reboot remote host "
|
||||
@ -108,25 +100,29 @@ class RebootHostOperation(tobiko.Operation):
|
||||
else:
|
||||
LOG.exception(f"Unable to reboot remote host: {ex}")
|
||||
raise RebootHostTimeoutError(
|
||||
hostname=self.hostname or ssh_client.host,
|
||||
hostname=self.hostname or self.ssh_client.host,
|
||||
timeout=attempt.timeout) from ex
|
||||
else:
|
||||
self.is_rebooted = False
|
||||
LOG.info(f"Host '{self.hostname}' is rebooting "
|
||||
f"(command='{self.command}').")
|
||||
break
|
||||
finally:
|
||||
# Ensure we close connection after rebooting command
|
||||
ssh_client.close()
|
||||
|
||||
if self.wait:
|
||||
self.wait_for_operation()
|
||||
self.ssh_client.close()
|
||||
|
||||
def cleanup_fixture(self):
|
||||
self.is_rebooted = None
|
||||
self.hostname = None
|
||||
self.is_rebooted = False
|
||||
self.start_time = None
|
||||
|
||||
@property
|
||||
def command(self) -> _command.ShellCommand:
|
||||
return _command.shell_command(
|
||||
['sudo', '/bin/sh', '-c', self.method.command])
|
||||
|
||||
@property
|
||||
def hostname(self) -> str:
|
||||
return self.ssh_client.hostname
|
||||
|
||||
@property
|
||||
def elapsed_time(self) -> tobiko.Seconds:
|
||||
if self.start_time is None:
|
||||
@ -166,7 +162,6 @@ class RebootHostOperation(tobiko.Operation):
|
||||
f"'{self.hostname}'", exc_info=1)
|
||||
attempt.check_limits()
|
||||
else:
|
||||
|
||||
# verify that reboot actually happened by comparing elapsed
|
||||
# time with up_time
|
||||
elapsed_time = self.elapsed_time
|
||||
@ -184,8 +179,4 @@ class RebootHostOperation(tobiko.Operation):
|
||||
attempt.check_limits()
|
||||
finally:
|
||||
if not self.is_rebooted:
|
||||
try:
|
||||
tobiko.cleanup_fixture(self.ssh_client)
|
||||
except Exception:
|
||||
LOG.exception("Error closing SSH connection to "
|
||||
f"'{self.hostname}'")
|
||||
self.ssh_client.close()
|
||||
|
@ -21,6 +21,7 @@ from oslo_log import log
|
||||
import testtools
|
||||
|
||||
import tobiko
|
||||
from tobiko.shell import ping
|
||||
from tobiko.shell import sh
|
||||
from tobiko.openstack import nova
|
||||
from tobiko.openstack import stacks
|
||||
@ -37,7 +38,7 @@ class RebootHostTest(testtools.TestCase):
|
||||
|
||||
stack = tobiko.required_setup_fixture(RebootHostStack)
|
||||
|
||||
def test_reboot_host(self, **params):
|
||||
def test_reboot_host(self, nova_reboot=False, **params):
|
||||
server = self.stack.ensure_server_status('ACTIVE')
|
||||
self.assertEqual('ACTIVE', server.status)
|
||||
|
||||
@ -53,19 +54,16 @@ class RebootHostTest(testtools.TestCase):
|
||||
timeout=90.)
|
||||
|
||||
reboot = sh.reboot_host(ssh_client=ssh_client, **params)
|
||||
|
||||
self.assertIs(ssh_client, reboot.ssh_client)
|
||||
self.assertEqual(ssh_client.hostname, reboot.hostname)
|
||||
self.assertIs(params.get('wait', True), reboot.wait)
|
||||
hard = params.get('hard', False)
|
||||
command = (params.get('method') or
|
||||
(hard and sh.hard_reset_method) or
|
||||
sh.soft_reset_method)
|
||||
self.assertEqual(command, reboot.command)
|
||||
method = params.get('method') or sh.soft_reset_method
|
||||
self.assertIs(method, reboot.method)
|
||||
|
||||
if not reboot.wait:
|
||||
self.assertFalse(reboot.is_rebooted)
|
||||
if not reboot.is_rebooted:
|
||||
self.assert_is_not_connected(ssh_client)
|
||||
if nova_reboot:
|
||||
ping.ping_until_unreceived(self.stack.ip_address)
|
||||
nova.reboot_server(server)
|
||||
reboot.wait_for_operation()
|
||||
|
||||
self.assertTrue(reboot.is_rebooted)
|
||||
@ -80,8 +78,10 @@ class RebootHostTest(testtools.TestCase):
|
||||
"uptime=%r", uptime_1)
|
||||
self.assertGreater(boottime_1, boottime_0)
|
||||
|
||||
def test_reboot_host_with_hard(self):
|
||||
self.test_reboot_host(hard=True)
|
||||
def test_reboot_host_with_chash_method(self):
|
||||
self.test_reboot_host(method=sh.crash_method,
|
||||
wait=False,
|
||||
nova_reboot=True)
|
||||
|
||||
def test_reboot_host_with_hard_method(self):
|
||||
self.test_reboot_host(method=sh.hard_reset_method)
|
||||
@ -90,14 +90,11 @@ class RebootHostTest(testtools.TestCase):
|
||||
self.test_reboot_host(method=sh.soft_reset_method)
|
||||
|
||||
def test_reboot_host_with_invalid_method(self):
|
||||
self.assertRaises(ValueError,
|
||||
self.assertRaises(TypeError,
|
||||
sh.reboot_host,
|
||||
ssh_client=self.stack.ssh_client,
|
||||
method='<invalid-method>')
|
||||
|
||||
def test_reboot_host_with_no_hard(self):
|
||||
self.test_reboot_host(hard=False)
|
||||
|
||||
def test_reboot_host_with_wait(self):
|
||||
self.test_reboot_host(wait=True)
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user