diff --git a/tobiko/shell/sh/__init__.py b/tobiko/shell/sh/__init__.py index ff6f0c75c..acbabce73 100644 --- a/tobiko/shell/sh/__init__.py +++ b/tobiko/shell/sh/__init__.py @@ -22,7 +22,9 @@ from tobiko.shell.sh import _hostname from tobiko.shell.sh import _io from tobiko.shell.sh import _local from tobiko.shell.sh import _process +from tobiko.shell.sh import _reboot from tobiko.shell.sh import _ssh +from tobiko.shell.sh import _uptime shell_command = _command.shell_command @@ -51,6 +53,10 @@ LocalExecutePathFixture = _local.LocalExecutePathFixture process = _process.process ShellProcessFixture = _process.ShellProcessFixture +reboot_host = _reboot.reboot_host + ssh_process = _ssh.ssh_process ssh_execute = _ssh.ssh_execute SSHShellProcessFixture = _ssh.SSHShellProcessFixture + +get_uptime = _uptime.get_uptime diff --git a/tobiko/shell/sh/_reboot.py b/tobiko/shell/sh/_reboot.py new file mode 100644 index 000000000..2a7e58f4e --- /dev/null +++ b/tobiko/shell/sh/_reboot.py @@ -0,0 +1,134 @@ +# Copyright 2019 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import time + +from oslo_log import log + +import tobiko +from tobiko.shell.sh import _execute +from tobiko.shell.sh import _hostname +from tobiko.shell.sh import _uptime + + +LOG = log.getLogger(__name__) + + +class RebootHostTimeoutError(tobiko.TobikoException): + message = "host {hostname!r} not rebooted after {timeout!s} seconds" + + +def reboot_host(ssh_client, wait=True, timeout=None, sleep_interval=None, + retry_interval=None): + """Gracefully reboots a remote host using an SSH client + + Given an SSH client to a remote host it executes /sbin/reboot command + and then it start polling for remote host uptime value to make sure + the node is actually rebooted before a given timeout. + """ + + with ssh_client: + hostname = _hostname.get_hostname(ssh_client=ssh_client, + timeout=timeout) + LOG.debug('Rebooting host %r...', hostname) + _execute.execute('sudo /sbin/reboot', timeout=timeout, stdout=False, + ssh_client=ssh_client) + + if wait: + if timeout is None: + timeout = 300. + if sleep_interval is None: + sleep_interval = 1. + if retry_interval is None: + retry_interval = 100. + else: + retry_interval = max(retry_interval, 5.) + + start_time = time.time() + elapsed_time = 0. + retry_time = retry_interval + + while True: + try: + _wait_for_host_rebooted(ssh_client=ssh_client, + hostname=hostname, + start_time=start_time, + timeout=min(retry_time, timeout), + sleep_interval=sleep_interval) + break + + except RebootHostTimeoutError: + elapsed_time = time.time() - start_time + if elapsed_time >= timeout: + raise + + LOG.debug("Retrying rebooting host %r %s seconds after " + "reboot...", hostname, elapsed_time) + with ssh_client: + _execute.execute('sudo /sbin/reboot', timeout=( + timeout - elapsed_time), ssh_client=ssh_client) + elapsed_time = time.time() - start_time + retry_time = elapsed_time + retry_interval + + +def _wait_for_host_rebooted(ssh_client, hostname, start_time, timeout, + sleep_interval): + while not _is_host_rebooted(ssh_client=ssh_client, + hostname=hostname, + start_time=start_time, + timeout=timeout): + if sleep_interval > 0.: + time.sleep(sleep_interval) + + +def _is_host_rebooted(ssh_client, hostname, start_time, timeout): + # ensure SSH connection is closed before retrying connecting + tobiko.cleanup_fixture(ssh_client) + assert ssh_client.client is None + + elapsed_time = time.time() - start_time + if elapsed_time >= timeout: + raise RebootHostTimeoutError(hostname=hostname, + timeout=timeout) + + LOG.debug("Reconnecting to host %r %s seconds after reboot...", + hostname, elapsed_time) + try: + uptime = _uptime.get_uptime(ssh_client=ssh_client, + timeout=(timeout-elapsed_time)) + except Exception as ex: + # if disconnected while getting uptime we assume the VM is just + # rebooting. These are good news! + tobiko.cleanup_fixture(ssh_client) + assert ssh_client.client is None + elapsed_time = time.time() - start_time + LOG.debug("Unable to get uptime from %r host after %r " + "seconds: %s", hostname, elapsed_time, ex) + return False + + # verify that reboot actually happened by comparing elapsed time with + # uptime + elapsed_time = time.time() - start_time + if uptime >= elapsed_time: + tobiko.cleanup_fixture(ssh_client) + assert ssh_client.client is None + LOG.warning("Host %r still not rebooted after %s seconds after reboot " + "(uptime=%r)", hostname, elapsed_time, uptime) + return False + + LOG.debug("Reconnected to host %r %s seconds after reboot " + "(uptime=%r)", hostname, elapsed_time, uptime) + assert ssh_client.client is not None + return True diff --git a/tobiko/shell/sh/_uptime.py b/tobiko/shell/sh/_uptime.py new file mode 100644 index 000000000..13779e7aa --- /dev/null +++ b/tobiko/shell/sh/_uptime.py @@ -0,0 +1,42 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import tobiko +from tobiko.shell.sh import _execute + + +class UptimeError(tobiko.TobikoException): + message = "Unable to get uptime from host: {error}" + + +def get_uptime(**execute_params): + """Returns the number of seconds passed since last host reboot + + It reads and parses remote special file /proc/uptime and returns a floating + point value that represents the number of seconds passed since last host + reboot + """ + result = _execute.execute('cat /proc/uptime', stdin=False, stdout=True, + stderr=True, expect_exit_status=None, + **execute_params) + output = result.stdout and result.stdout.strip() + if result.exit_status or not output: + raise UptimeError(error=result.stderr) + + uptime_line = output.splitlines()[0] + uptime_string = uptime_line.split()[0] + return float(uptime_string) diff --git a/tobiko/tests/functional/shell/test_reboot.py b/tobiko/tests/functional/shell/test_reboot.py new file mode 100644 index 000000000..7d01e426b --- /dev/null +++ b/tobiko/tests/functional/shell/test_reboot.py @@ -0,0 +1,92 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import time + +from oslo_log import log +import testtools + +import tobiko +from tobiko.shell import sh +from tobiko.openstack import nova +from tobiko.openstack import stacks + + +LOG = log.getLogger(__name__) + + +class RebootableServer(stacks.CirrosServerStackFixture): + "Server to be rebooted" + + +class RebootHostTest(testtools.TestCase): + + stack = tobiko.required_setup_fixture(RebootableServer) + + def test_reboot_host(self, **params): + server = nova.activate_server(self.stack.server_id) + self.assertEqual('ACTIVE', server.status) + + ssh_client = self.stack.ssh_client + uptime_0 = sh.get_uptime(ssh_client=ssh_client) + LOG.debug("Testing reboot command on remote host: " + "uptime=%r", uptime_0) + boottime_0 = time.time() - uptime_0 + + sh.reboot_host(ssh_client=ssh_client, **params) + + server = nova.wait_for_server_status(server, 'ACTIVE') + self.assertEqual('ACTIVE', server.status) + + wait = params.get('wait', True) + if wait: + self.assert_is_connected(ssh_client) + uptime_1 = sh.get_uptime(ssh_client=ssh_client) + boottime_1 = time.time() - uptime_1 + LOG.debug("Reboot operation executed on remote host: " + "uptime=%r", uptime_1) + self.assertGreater(boottime_1, boottime_0) + else: + self.assert_is_not_connected(ssh_client) + + def test_reboot_host_with_wait(self): + self.test_reboot_host(wait=True) + + def test_reboot_host_with_no_wait(self): + self.test_reboot_host(wait=False) + + def test_reboot_server_after_shutoff(self): + server = nova.activate_server(self.stack.server_id) + self.assertEqual('ACTIVE', server.status) + ssh_client = self.stack.ssh_client + ssh_client.connect() + self.assert_is_connected(ssh_client) + + server = nova.shutoff_server(self.stack.server_id) + self.assertEqual('SHUTOFF', server.status) + + self.assertRaises(sh.HostNameError, sh.reboot_host, + ssh_client=ssh_client, timeout=5.0) + self.assert_is_not_connected(ssh_client) + server = nova.wait_for_server_status(self.stack.server_id, 'SHUTOFF') + self.assertEqual('SHUTOFF', server.status) + + def assert_is_connected(self, ssh_client): + self.assertIsNotNone(ssh_client.client) + + def assert_is_not_connected(self, ssh_client): + self.assertIsNone(ssh_client.client)