Create shell function to reboot an host
This adds a new function called reboot_host such that given an SSH client to a remote host it executes /sbin/reboot command and then it start polling for remote host uptime value to make sure the node is actually rebooted before a given timeout. Change-Id: I4ec2954484d6f8cc11e85f6994c174c427ec2c16
This commit is contained in:
parent
91adc50f8d
commit
9c616e2c7a
|
@ -22,7 +22,9 @@ from tobiko.shell.sh import _hostname
|
||||||
from tobiko.shell.sh import _io
|
from tobiko.shell.sh import _io
|
||||||
from tobiko.shell.sh import _local
|
from tobiko.shell.sh import _local
|
||||||
from tobiko.shell.sh import _process
|
from tobiko.shell.sh import _process
|
||||||
|
from tobiko.shell.sh import _reboot
|
||||||
from tobiko.shell.sh import _ssh
|
from tobiko.shell.sh import _ssh
|
||||||
|
from tobiko.shell.sh import _uptime
|
||||||
|
|
||||||
|
|
||||||
shell_command = _command.shell_command
|
shell_command = _command.shell_command
|
||||||
|
@ -51,6 +53,10 @@ LocalExecutePathFixture = _local.LocalExecutePathFixture
|
||||||
process = _process.process
|
process = _process.process
|
||||||
ShellProcessFixture = _process.ShellProcessFixture
|
ShellProcessFixture = _process.ShellProcessFixture
|
||||||
|
|
||||||
|
reboot_host = _reboot.reboot_host
|
||||||
|
|
||||||
ssh_process = _ssh.ssh_process
|
ssh_process = _ssh.ssh_process
|
||||||
ssh_execute = _ssh.ssh_execute
|
ssh_execute = _ssh.ssh_execute
|
||||||
SSHShellProcessFixture = _ssh.SSHShellProcessFixture
|
SSHShellProcessFixture = _ssh.SSHShellProcessFixture
|
||||||
|
|
||||||
|
get_uptime = _uptime.get_uptime
|
||||||
|
|
|
@ -0,0 +1,134 @@
|
||||||
|
# Copyright 2019 Red Hat
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
from __future__ import absolute_import
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
|
from oslo_log import log
|
||||||
|
|
||||||
|
import tobiko
|
||||||
|
from tobiko.shell.sh import _execute
|
||||||
|
from tobiko.shell.sh import _hostname
|
||||||
|
from tobiko.shell.sh import _uptime
|
||||||
|
|
||||||
|
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class RebootHostTimeoutError(tobiko.TobikoException):
|
||||||
|
message = "host {hostname!r} not rebooted after {timeout!s} seconds"
|
||||||
|
|
||||||
|
|
||||||
|
def reboot_host(ssh_client, wait=True, timeout=None, sleep_interval=None,
|
||||||
|
retry_interval=None):
|
||||||
|
"""Gracefully reboots a remote host using an SSH client
|
||||||
|
|
||||||
|
Given an SSH client to a remote host it executes /sbin/reboot command
|
||||||
|
and then it start polling for remote host uptime value to make sure
|
||||||
|
the node is actually rebooted before a given timeout.
|
||||||
|
"""
|
||||||
|
|
||||||
|
with ssh_client:
|
||||||
|
hostname = _hostname.get_hostname(ssh_client=ssh_client,
|
||||||
|
timeout=timeout)
|
||||||
|
LOG.debug('Rebooting host %r...', hostname)
|
||||||
|
_execute.execute('sudo /sbin/reboot', timeout=timeout, stdout=False,
|
||||||
|
ssh_client=ssh_client)
|
||||||
|
|
||||||
|
if wait:
|
||||||
|
if timeout is None:
|
||||||
|
timeout = 300.
|
||||||
|
if sleep_interval is None:
|
||||||
|
sleep_interval = 1.
|
||||||
|
if retry_interval is None:
|
||||||
|
retry_interval = 100.
|
||||||
|
else:
|
||||||
|
retry_interval = max(retry_interval, 5.)
|
||||||
|
|
||||||
|
start_time = time.time()
|
||||||
|
elapsed_time = 0.
|
||||||
|
retry_time = retry_interval
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
_wait_for_host_rebooted(ssh_client=ssh_client,
|
||||||
|
hostname=hostname,
|
||||||
|
start_time=start_time,
|
||||||
|
timeout=min(retry_time, timeout),
|
||||||
|
sleep_interval=sleep_interval)
|
||||||
|
break
|
||||||
|
|
||||||
|
except RebootHostTimeoutError:
|
||||||
|
elapsed_time = time.time() - start_time
|
||||||
|
if elapsed_time >= timeout:
|
||||||
|
raise
|
||||||
|
|
||||||
|
LOG.debug("Retrying rebooting host %r %s seconds after "
|
||||||
|
"reboot...", hostname, elapsed_time)
|
||||||
|
with ssh_client:
|
||||||
|
_execute.execute('sudo /sbin/reboot', timeout=(
|
||||||
|
timeout - elapsed_time), ssh_client=ssh_client)
|
||||||
|
elapsed_time = time.time() - start_time
|
||||||
|
retry_time = elapsed_time + retry_interval
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_host_rebooted(ssh_client, hostname, start_time, timeout,
|
||||||
|
sleep_interval):
|
||||||
|
while not _is_host_rebooted(ssh_client=ssh_client,
|
||||||
|
hostname=hostname,
|
||||||
|
start_time=start_time,
|
||||||
|
timeout=timeout):
|
||||||
|
if sleep_interval > 0.:
|
||||||
|
time.sleep(sleep_interval)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_host_rebooted(ssh_client, hostname, start_time, timeout):
|
||||||
|
# ensure SSH connection is closed before retrying connecting
|
||||||
|
tobiko.cleanup_fixture(ssh_client)
|
||||||
|
assert ssh_client.client is None
|
||||||
|
|
||||||
|
elapsed_time = time.time() - start_time
|
||||||
|
if elapsed_time >= timeout:
|
||||||
|
raise RebootHostTimeoutError(hostname=hostname,
|
||||||
|
timeout=timeout)
|
||||||
|
|
||||||
|
LOG.debug("Reconnecting to host %r %s seconds after reboot...",
|
||||||
|
hostname, elapsed_time)
|
||||||
|
try:
|
||||||
|
uptime = _uptime.get_uptime(ssh_client=ssh_client,
|
||||||
|
timeout=(timeout-elapsed_time))
|
||||||
|
except Exception as ex:
|
||||||
|
# if disconnected while getting uptime we assume the VM is just
|
||||||
|
# rebooting. These are good news!
|
||||||
|
tobiko.cleanup_fixture(ssh_client)
|
||||||
|
assert ssh_client.client is None
|
||||||
|
elapsed_time = time.time() - start_time
|
||||||
|
LOG.debug("Unable to get uptime from %r host after %r "
|
||||||
|
"seconds: %s", hostname, elapsed_time, ex)
|
||||||
|
return False
|
||||||
|
|
||||||
|
# verify that reboot actually happened by comparing elapsed time with
|
||||||
|
# uptime
|
||||||
|
elapsed_time = time.time() - start_time
|
||||||
|
if uptime >= elapsed_time:
|
||||||
|
tobiko.cleanup_fixture(ssh_client)
|
||||||
|
assert ssh_client.client is None
|
||||||
|
LOG.warning("Host %r still not rebooted after %s seconds after reboot "
|
||||||
|
"(uptime=%r)", hostname, elapsed_time, uptime)
|
||||||
|
return False
|
||||||
|
|
||||||
|
LOG.debug("Reconnected to host %r %s seconds after reboot "
|
||||||
|
"(uptime=%r)", hostname, elapsed_time, uptime)
|
||||||
|
assert ssh_client.client is not None
|
||||||
|
return True
|
|
@ -0,0 +1,42 @@
|
||||||
|
# Copyright (c) 2019 Red Hat, Inc.
|
||||||
|
#
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
from __future__ import absolute_import
|
||||||
|
|
||||||
|
import tobiko
|
||||||
|
from tobiko.shell.sh import _execute
|
||||||
|
|
||||||
|
|
||||||
|
class UptimeError(tobiko.TobikoException):
|
||||||
|
message = "Unable to get uptime from host: {error}"
|
||||||
|
|
||||||
|
|
||||||
|
def get_uptime(**execute_params):
|
||||||
|
"""Returns the number of seconds passed since last host reboot
|
||||||
|
|
||||||
|
It reads and parses remote special file /proc/uptime and returns a floating
|
||||||
|
point value that represents the number of seconds passed since last host
|
||||||
|
reboot
|
||||||
|
"""
|
||||||
|
result = _execute.execute('cat /proc/uptime', stdin=False, stdout=True,
|
||||||
|
stderr=True, expect_exit_status=None,
|
||||||
|
**execute_params)
|
||||||
|
output = result.stdout and result.stdout.strip()
|
||||||
|
if result.exit_status or not output:
|
||||||
|
raise UptimeError(error=result.stderr)
|
||||||
|
|
||||||
|
uptime_line = output.splitlines()[0]
|
||||||
|
uptime_string = uptime_line.split()[0]
|
||||||
|
return float(uptime_string)
|
|
@ -0,0 +1,92 @@
|
||||||
|
# Copyright (c) 2019 Red Hat, Inc.
|
||||||
|
#
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
from __future__ import absolute_import
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
|
from oslo_log import log
|
||||||
|
import testtools
|
||||||
|
|
||||||
|
import tobiko
|
||||||
|
from tobiko.shell import sh
|
||||||
|
from tobiko.openstack import nova
|
||||||
|
from tobiko.openstack import stacks
|
||||||
|
|
||||||
|
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class RebootableServer(stacks.CirrosServerStackFixture):
|
||||||
|
"Server to be rebooted"
|
||||||
|
|
||||||
|
|
||||||
|
class RebootHostTest(testtools.TestCase):
|
||||||
|
|
||||||
|
stack = tobiko.required_setup_fixture(RebootableServer)
|
||||||
|
|
||||||
|
def test_reboot_host(self, **params):
|
||||||
|
server = nova.activate_server(self.stack.server_id)
|
||||||
|
self.assertEqual('ACTIVE', server.status)
|
||||||
|
|
||||||
|
ssh_client = self.stack.ssh_client
|
||||||
|
uptime_0 = sh.get_uptime(ssh_client=ssh_client)
|
||||||
|
LOG.debug("Testing reboot command on remote host: "
|
||||||
|
"uptime=%r", uptime_0)
|
||||||
|
boottime_0 = time.time() - uptime_0
|
||||||
|
|
||||||
|
sh.reboot_host(ssh_client=ssh_client, **params)
|
||||||
|
|
||||||
|
server = nova.wait_for_server_status(server, 'ACTIVE')
|
||||||
|
self.assertEqual('ACTIVE', server.status)
|
||||||
|
|
||||||
|
wait = params.get('wait', True)
|
||||||
|
if wait:
|
||||||
|
self.assert_is_connected(ssh_client)
|
||||||
|
uptime_1 = sh.get_uptime(ssh_client=ssh_client)
|
||||||
|
boottime_1 = time.time() - uptime_1
|
||||||
|
LOG.debug("Reboot operation executed on remote host: "
|
||||||
|
"uptime=%r", uptime_1)
|
||||||
|
self.assertGreater(boottime_1, boottime_0)
|
||||||
|
else:
|
||||||
|
self.assert_is_not_connected(ssh_client)
|
||||||
|
|
||||||
|
def test_reboot_host_with_wait(self):
|
||||||
|
self.test_reboot_host(wait=True)
|
||||||
|
|
||||||
|
def test_reboot_host_with_no_wait(self):
|
||||||
|
self.test_reboot_host(wait=False)
|
||||||
|
|
||||||
|
def test_reboot_server_after_shutoff(self):
|
||||||
|
server = nova.activate_server(self.stack.server_id)
|
||||||
|
self.assertEqual('ACTIVE', server.status)
|
||||||
|
ssh_client = self.stack.ssh_client
|
||||||
|
ssh_client.connect()
|
||||||
|
self.assert_is_connected(ssh_client)
|
||||||
|
|
||||||
|
server = nova.shutoff_server(self.stack.server_id)
|
||||||
|
self.assertEqual('SHUTOFF', server.status)
|
||||||
|
|
||||||
|
self.assertRaises(sh.HostNameError, sh.reboot_host,
|
||||||
|
ssh_client=ssh_client, timeout=5.0)
|
||||||
|
self.assert_is_not_connected(ssh_client)
|
||||||
|
server = nova.wait_for_server_status(self.stack.server_id, 'SHUTOFF')
|
||||||
|
self.assertEqual('SHUTOFF', server.status)
|
||||||
|
|
||||||
|
def assert_is_connected(self, ssh_client):
|
||||||
|
self.assertIsNotNone(ssh_client.client)
|
||||||
|
|
||||||
|
def assert_is_not_connected(self, ssh_client):
|
||||||
|
self.assertIsNone(ssh_client.client)
|
Loading…
Reference in New Issue