Create shell function to reboot an host

This adds a new function called reboot_host such that
given an SSH client to a remote host it executes /sbin/reboot command
and then it start polling for remote host uptime value to make sure
the node is actually rebooted before a given timeout.

Change-Id: I4ec2954484d6f8cc11e85f6994c174c427ec2c16
This commit is contained in:
Federico Ressi 2020-01-22 16:54:21 +01:00
parent 91adc50f8d
commit 9c616e2c7a
4 changed files with 274 additions and 0 deletions

View File

@ -22,7 +22,9 @@ from tobiko.shell.sh import _hostname
from tobiko.shell.sh import _io
from tobiko.shell.sh import _local
from tobiko.shell.sh import _process
from tobiko.shell.sh import _reboot
from tobiko.shell.sh import _ssh
from tobiko.shell.sh import _uptime
shell_command = _command.shell_command
@ -51,6 +53,10 @@ LocalExecutePathFixture = _local.LocalExecutePathFixture
process = _process.process
ShellProcessFixture = _process.ShellProcessFixture
reboot_host = _reboot.reboot_host
ssh_process = _ssh.ssh_process
ssh_execute = _ssh.ssh_execute
SSHShellProcessFixture = _ssh.SSHShellProcessFixture
get_uptime = _uptime.get_uptime

134
tobiko/shell/sh/_reboot.py Normal file
View File

@ -0,0 +1,134 @@
# Copyright 2019 Red Hat
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import time
from oslo_log import log
import tobiko
from tobiko.shell.sh import _execute
from tobiko.shell.sh import _hostname
from tobiko.shell.sh import _uptime
LOG = log.getLogger(__name__)
class RebootHostTimeoutError(tobiko.TobikoException):
message = "host {hostname!r} not rebooted after {timeout!s} seconds"
def reboot_host(ssh_client, wait=True, timeout=None, sleep_interval=None,
retry_interval=None):
"""Gracefully reboots a remote host using an SSH client
Given an SSH client to a remote host it executes /sbin/reboot command
and then it start polling for remote host uptime value to make sure
the node is actually rebooted before a given timeout.
"""
with ssh_client:
hostname = _hostname.get_hostname(ssh_client=ssh_client,
timeout=timeout)
LOG.debug('Rebooting host %r...', hostname)
_execute.execute('sudo /sbin/reboot', timeout=timeout, stdout=False,
ssh_client=ssh_client)
if wait:
if timeout is None:
timeout = 300.
if sleep_interval is None:
sleep_interval = 1.
if retry_interval is None:
retry_interval = 100.
else:
retry_interval = max(retry_interval, 5.)
start_time = time.time()
elapsed_time = 0.
retry_time = retry_interval
while True:
try:
_wait_for_host_rebooted(ssh_client=ssh_client,
hostname=hostname,
start_time=start_time,
timeout=min(retry_time, timeout),
sleep_interval=sleep_interval)
break
except RebootHostTimeoutError:
elapsed_time = time.time() - start_time
if elapsed_time >= timeout:
raise
LOG.debug("Retrying rebooting host %r %s seconds after "
"reboot...", hostname, elapsed_time)
with ssh_client:
_execute.execute('sudo /sbin/reboot', timeout=(
timeout - elapsed_time), ssh_client=ssh_client)
elapsed_time = time.time() - start_time
retry_time = elapsed_time + retry_interval
def _wait_for_host_rebooted(ssh_client, hostname, start_time, timeout,
sleep_interval):
while not _is_host_rebooted(ssh_client=ssh_client,
hostname=hostname,
start_time=start_time,
timeout=timeout):
if sleep_interval > 0.:
time.sleep(sleep_interval)
def _is_host_rebooted(ssh_client, hostname, start_time, timeout):
# ensure SSH connection is closed before retrying connecting
tobiko.cleanup_fixture(ssh_client)
assert ssh_client.client is None
elapsed_time = time.time() - start_time
if elapsed_time >= timeout:
raise RebootHostTimeoutError(hostname=hostname,
timeout=timeout)
LOG.debug("Reconnecting to host %r %s seconds after reboot...",
hostname, elapsed_time)
try:
uptime = _uptime.get_uptime(ssh_client=ssh_client,
timeout=(timeout-elapsed_time))
except Exception as ex:
# if disconnected while getting uptime we assume the VM is just
# rebooting. These are good news!
tobiko.cleanup_fixture(ssh_client)
assert ssh_client.client is None
elapsed_time = time.time() - start_time
LOG.debug("Unable to get uptime from %r host after %r "
"seconds: %s", hostname, elapsed_time, ex)
return False
# verify that reboot actually happened by comparing elapsed time with
# uptime
elapsed_time = time.time() - start_time
if uptime >= elapsed_time:
tobiko.cleanup_fixture(ssh_client)
assert ssh_client.client is None
LOG.warning("Host %r still not rebooted after %s seconds after reboot "
"(uptime=%r)", hostname, elapsed_time, uptime)
return False
LOG.debug("Reconnected to host %r %s seconds after reboot "
"(uptime=%r)", hostname, elapsed_time, uptime)
assert ssh_client.client is not None
return True

View File

@ -0,0 +1,42 @@
# Copyright (c) 2019 Red Hat, Inc.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import tobiko
from tobiko.shell.sh import _execute
class UptimeError(tobiko.TobikoException):
message = "Unable to get uptime from host: {error}"
def get_uptime(**execute_params):
"""Returns the number of seconds passed since last host reboot
It reads and parses remote special file /proc/uptime and returns a floating
point value that represents the number of seconds passed since last host
reboot
"""
result = _execute.execute('cat /proc/uptime', stdin=False, stdout=True,
stderr=True, expect_exit_status=None,
**execute_params)
output = result.stdout and result.stdout.strip()
if result.exit_status or not output:
raise UptimeError(error=result.stderr)
uptime_line = output.splitlines()[0]
uptime_string = uptime_line.split()[0]
return float(uptime_string)

View File

@ -0,0 +1,92 @@
# Copyright (c) 2019 Red Hat, Inc.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import time
from oslo_log import log
import testtools
import tobiko
from tobiko.shell import sh
from tobiko.openstack import nova
from tobiko.openstack import stacks
LOG = log.getLogger(__name__)
class RebootableServer(stacks.CirrosServerStackFixture):
"Server to be rebooted"
class RebootHostTest(testtools.TestCase):
stack = tobiko.required_setup_fixture(RebootableServer)
def test_reboot_host(self, **params):
server = nova.activate_server(self.stack.server_id)
self.assertEqual('ACTIVE', server.status)
ssh_client = self.stack.ssh_client
uptime_0 = sh.get_uptime(ssh_client=ssh_client)
LOG.debug("Testing reboot command on remote host: "
"uptime=%r", uptime_0)
boottime_0 = time.time() - uptime_0
sh.reboot_host(ssh_client=ssh_client, **params)
server = nova.wait_for_server_status(server, 'ACTIVE')
self.assertEqual('ACTIVE', server.status)
wait = params.get('wait', True)
if wait:
self.assert_is_connected(ssh_client)
uptime_1 = sh.get_uptime(ssh_client=ssh_client)
boottime_1 = time.time() - uptime_1
LOG.debug("Reboot operation executed on remote host: "
"uptime=%r", uptime_1)
self.assertGreater(boottime_1, boottime_0)
else:
self.assert_is_not_connected(ssh_client)
def test_reboot_host_with_wait(self):
self.test_reboot_host(wait=True)
def test_reboot_host_with_no_wait(self):
self.test_reboot_host(wait=False)
def test_reboot_server_after_shutoff(self):
server = nova.activate_server(self.stack.server_id)
self.assertEqual('ACTIVE', server.status)
ssh_client = self.stack.ssh_client
ssh_client.connect()
self.assert_is_connected(ssh_client)
server = nova.shutoff_server(self.stack.server_id)
self.assertEqual('SHUTOFF', server.status)
self.assertRaises(sh.HostNameError, sh.reboot_host,
ssh_client=ssh_client, timeout=5.0)
self.assert_is_not_connected(ssh_client)
server = nova.wait_for_server_status(self.stack.server_id, 'SHUTOFF')
self.assertEqual('SHUTOFF', server.status)
def assert_is_connected(self, ssh_client):
self.assertIsNotNone(ssh_client.client)
def assert_is_not_connected(self, ssh_client):
self.assertIsNone(ssh_client.client)