Create shell function to reboot an host
This adds a new function called reboot_host such that given an SSH client to a remote host it executes /sbin/reboot command and then it start polling for remote host uptime value to make sure the node is actually rebooted before a given timeout. Change-Id: I4ec2954484d6f8cc11e85f6994c174c427ec2c16
This commit is contained in:
Federico Ressi
parent
91adc50f8d
commit
9c616e2c7a
@ -22,7 +22,9 @@ from tobiko.shell.sh import _hostname
|
||||
from tobiko.shell.sh import _io
|
||||
from tobiko.shell.sh import _local
|
||||
from tobiko.shell.sh import _process
|
||||
from tobiko.shell.sh import _reboot
|
||||
from tobiko.shell.sh import _ssh
|
||||
from tobiko.shell.sh import _uptime
|
||||
|
||||
|
||||
shell_command = _command.shell_command
|
||||
@ -51,6 +53,10 @@ LocalExecutePathFixture = _local.LocalExecutePathFixture
|
||||
process = _process.process
|
||||
ShellProcessFixture = _process.ShellProcessFixture
|
||||
|
||||
reboot_host = _reboot.reboot_host
|
||||
|
||||
ssh_process = _ssh.ssh_process
|
||||
ssh_execute = _ssh.ssh_execute
|
||||
SSHShellProcessFixture = _ssh.SSHShellProcessFixture
|
||||
|
||||
get_uptime = _uptime.get_uptime
|
||||
|
||||
134
tobiko/shell/sh/_reboot.py
Normal file
134
tobiko/shell/sh/_reboot.py
Normal file
@ -0,0 +1,134 @@
|
||||
# Copyright 2019 Red Hat
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import absolute_import
|
||||
|
||||
import time
|
||||
|
||||
from oslo_log import log
|
||||
|
||||
import tobiko
|
||||
from tobiko.shell.sh import _execute
|
||||
from tobiko.shell.sh import _hostname
|
||||
from tobiko.shell.sh import _uptime
|
||||
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class RebootHostTimeoutError(tobiko.TobikoException):
|
||||
message = "host {hostname!r} not rebooted after {timeout!s} seconds"
|
||||
|
||||
|
||||
def reboot_host(ssh_client, wait=True, timeout=None, sleep_interval=None,
|
||||
retry_interval=None):
|
||||
"""Gracefully reboots a remote host using an SSH client
|
||||
|
||||
Given an SSH client to a remote host it executes /sbin/reboot command
|
||||
and then it start polling for remote host uptime value to make sure
|
||||
the node is actually rebooted before a given timeout.
|
||||
"""
|
||||
|
||||
with ssh_client:
|
||||
hostname = _hostname.get_hostname(ssh_client=ssh_client,
|
||||
timeout=timeout)
|
||||
LOG.debug('Rebooting host %r...', hostname)
|
||||
_execute.execute('sudo /sbin/reboot', timeout=timeout, stdout=False,
|
||||
ssh_client=ssh_client)
|
||||
|
||||
if wait:
|
||||
if timeout is None:
|
||||
timeout = 300.
|
||||
if sleep_interval is None:
|
||||
sleep_interval = 1.
|
||||
if retry_interval is None:
|
||||
retry_interval = 100.
|
||||
else:
|
||||
retry_interval = max(retry_interval, 5.)
|
||||
|
||||
start_time = time.time()
|
||||
elapsed_time = 0.
|
||||
retry_time = retry_interval
|
||||
|
||||
while True:
|
||||
try:
|
||||
_wait_for_host_rebooted(ssh_client=ssh_client,
|
||||
hostname=hostname,
|
||||
start_time=start_time,
|
||||
timeout=min(retry_time, timeout),
|
||||
sleep_interval=sleep_interval)
|
||||
break
|
||||
|
||||
except RebootHostTimeoutError:
|
||||
elapsed_time = time.time() - start_time
|
||||
if elapsed_time >= timeout:
|
||||
raise
|
||||
|
||||
LOG.debug("Retrying rebooting host %r %s seconds after "
|
||||
"reboot...", hostname, elapsed_time)
|
||||
with ssh_client:
|
||||
_execute.execute('sudo /sbin/reboot', timeout=(
|
||||
timeout - elapsed_time), ssh_client=ssh_client)
|
||||
elapsed_time = time.time() - start_time
|
||||
retry_time = elapsed_time + retry_interval
|
||||
|
||||
|
||||
def _wait_for_host_rebooted(ssh_client, hostname, start_time, timeout,
|
||||
sleep_interval):
|
||||
while not _is_host_rebooted(ssh_client=ssh_client,
|
||||
hostname=hostname,
|
||||
start_time=start_time,
|
||||
timeout=timeout):
|
||||
if sleep_interval > 0.:
|
||||
time.sleep(sleep_interval)
|
||||
|
||||
|
||||
def _is_host_rebooted(ssh_client, hostname, start_time, timeout):
|
||||
# ensure SSH connection is closed before retrying connecting
|
||||
tobiko.cleanup_fixture(ssh_client)
|
||||
assert ssh_client.client is None
|
||||
|
||||
elapsed_time = time.time() - start_time
|
||||
if elapsed_time >= timeout:
|
||||
raise RebootHostTimeoutError(hostname=hostname,
|
||||
timeout=timeout)
|
||||
|
||||
LOG.debug("Reconnecting to host %r %s seconds after reboot...",
|
||||
hostname, elapsed_time)
|
||||
try:
|
||||
uptime = _uptime.get_uptime(ssh_client=ssh_client,
|
||||
timeout=(timeout-elapsed_time))
|
||||
except Exception as ex:
|
||||
# if disconnected while getting uptime we assume the VM is just
|
||||
# rebooting. These are good news!
|
||||
tobiko.cleanup_fixture(ssh_client)
|
||||
assert ssh_client.client is None
|
||||
elapsed_time = time.time() - start_time
|
||||
LOG.debug("Unable to get uptime from %r host after %r "
|
||||
"seconds: %s", hostname, elapsed_time, ex)
|
||||
return False
|
||||
|
||||
# verify that reboot actually happened by comparing elapsed time with
|
||||
# uptime
|
||||
elapsed_time = time.time() - start_time
|
||||
if uptime >= elapsed_time:
|
||||
tobiko.cleanup_fixture(ssh_client)
|
||||
assert ssh_client.client is None
|
||||
LOG.warning("Host %r still not rebooted after %s seconds after reboot "
|
||||
"(uptime=%r)", hostname, elapsed_time, uptime)
|
||||
return False
|
||||
|
||||
LOG.debug("Reconnected to host %r %s seconds after reboot "
|
||||
"(uptime=%r)", hostname, elapsed_time, uptime)
|
||||
assert ssh_client.client is not None
|
||||
return True
|
||||
42
tobiko/shell/sh/_uptime.py
Normal file
42
tobiko/shell/sh/_uptime.py
Normal file
@ -0,0 +1,42 @@
|
||||
# Copyright (c) 2019 Red Hat, Inc.
|
||||
#
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import absolute_import
|
||||
|
||||
import tobiko
|
||||
from tobiko.shell.sh import _execute
|
||||
|
||||
|
||||
class UptimeError(tobiko.TobikoException):
|
||||
message = "Unable to get uptime from host: {error}"
|
||||
|
||||
|
||||
def get_uptime(**execute_params):
|
||||
"""Returns the number of seconds passed since last host reboot
|
||||
|
||||
It reads and parses remote special file /proc/uptime and returns a floating
|
||||
point value that represents the number of seconds passed since last host
|
||||
reboot
|
||||
"""
|
||||
result = _execute.execute('cat /proc/uptime', stdin=False, stdout=True,
|
||||
stderr=True, expect_exit_status=None,
|
||||
**execute_params)
|
||||
output = result.stdout and result.stdout.strip()
|
||||
if result.exit_status or not output:
|
||||
raise UptimeError(error=result.stderr)
|
||||
|
||||
uptime_line = output.splitlines()[0]
|
||||
uptime_string = uptime_line.split()[0]
|
||||
return float(uptime_string)
|
||||
92
tobiko/tests/functional/shell/test_reboot.py
Normal file
92
tobiko/tests/functional/shell/test_reboot.py
Normal file
@ -0,0 +1,92 @@
|
||||
# Copyright (c) 2019 Red Hat, Inc.
|
||||
#
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import absolute_import
|
||||
|
||||
import time
|
||||
|
||||
from oslo_log import log
|
||||
import testtools
|
||||
|
||||
import tobiko
|
||||
from tobiko.shell import sh
|
||||
from tobiko.openstack import nova
|
||||
from tobiko.openstack import stacks
|
||||
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class RebootableServer(stacks.CirrosServerStackFixture):
|
||||
"Server to be rebooted"
|
||||
|
||||
|
||||
class RebootHostTest(testtools.TestCase):
|
||||
|
||||
stack = tobiko.required_setup_fixture(RebootableServer)
|
||||
|
||||
def test_reboot_host(self, **params):
|
||||
server = nova.activate_server(self.stack.server_id)
|
||||
self.assertEqual('ACTIVE', server.status)
|
||||
|
||||
ssh_client = self.stack.ssh_client
|
||||
uptime_0 = sh.get_uptime(ssh_client=ssh_client)
|
||||
LOG.debug("Testing reboot command on remote host: "
|
||||
"uptime=%r", uptime_0)
|
||||
boottime_0 = time.time() - uptime_0
|
||||
|
||||
sh.reboot_host(ssh_client=ssh_client, **params)
|
||||
|
||||
server = nova.wait_for_server_status(server, 'ACTIVE')
|
||||
self.assertEqual('ACTIVE', server.status)
|
||||
|
||||
wait = params.get('wait', True)
|
||||
if wait:
|
||||
self.assert_is_connected(ssh_client)
|
||||
uptime_1 = sh.get_uptime(ssh_client=ssh_client)
|
||||
boottime_1 = time.time() - uptime_1
|
||||
LOG.debug("Reboot operation executed on remote host: "
|
||||
"uptime=%r", uptime_1)
|
||||
self.assertGreater(boottime_1, boottime_0)
|
||||
else:
|
||||
self.assert_is_not_connected(ssh_client)
|
||||
|
||||
def test_reboot_host_with_wait(self):
|
||||
self.test_reboot_host(wait=True)
|
||||
|
||||
def test_reboot_host_with_no_wait(self):
|
||||
self.test_reboot_host(wait=False)
|
||||
|
||||
def test_reboot_server_after_shutoff(self):
|
||||
server = nova.activate_server(self.stack.server_id)
|
||||
self.assertEqual('ACTIVE', server.status)
|
||||
ssh_client = self.stack.ssh_client
|
||||
ssh_client.connect()
|
||||
self.assert_is_connected(ssh_client)
|
||||
|
||||
server = nova.shutoff_server(self.stack.server_id)
|
||||
self.assertEqual('SHUTOFF', server.status)
|
||||
|
||||
self.assertRaises(sh.HostNameError, sh.reboot_host,
|
||||
ssh_client=ssh_client, timeout=5.0)
|
||||
self.assert_is_not_connected(ssh_client)
|
||||
server = nova.wait_for_server_status(self.stack.server_id, 'SHUTOFF')
|
||||
self.assertEqual('SHUTOFF', server.status)
|
||||
|
||||
def assert_is_connected(self, ssh_client):
|
||||
self.assertIsNotNone(ssh_client.client)
|
||||
|
||||
def assert_is_not_connected(self, ssh_client):
|
||||
self.assertIsNone(ssh_client.client)
|
||||
Loading…
x
Reference in New Issue
Block a user