From 4bdcc3b9af9bbce3c51bacda8282de75601f13c0 Mon Sep 17 00:00:00 2001 From: Ksenia Demina Date: Thu, 21 Apr 2016 11:11:52 +0300 Subject: [PATCH] Use the ssh_manager in cic_mm mode tests Change-Id: Iae925cd47b5586f09ecd5493e973447838e5fe9d Closes-Bug: 1572574 (cherry picked from commit 606dbe7cdfcae73ee0301faae8bfb87d2b4e7c9c) --- .../test_cic_maintenance_mode.py | 252 ++++++++++-------- 1 file changed, 134 insertions(+), 118 deletions(-) diff --git a/fuelweb_test/tests/tests_strength/test_cic_maintenance_mode.py b/fuelweb_test/tests/tests_strength/test_cic_maintenance_mode.py index b5c2a65e8..82cde1065 100644 --- a/fuelweb_test/tests/tests_strength/test_cic_maintenance_mode.py +++ b/fuelweb_test/tests/tests_strength/test_cic_maintenance_mode.py @@ -15,15 +15,11 @@ import time from devops.error import TimeoutError from devops.helpers.helpers import _wait -from devops.helpers.helpers import tcp_ping from devops.helpers.helpers import wait -from proboscis.asserts import assert_equal -from proboscis.asserts import assert_false -from proboscis.asserts import assert_true +from proboscis import asserts from proboscis import test -from fuelweb_test.helpers.checkers import check_auto_mode -from fuelweb_test.helpers.checkers import check_available_mode +from fuelweb_test.helpers import checkers from fuelweb_test.helpers.decorators import log_snapshot_after_test from fuelweb_test import logger from fuelweb_test import ostf_test_mapping @@ -34,7 +30,7 @@ from fuelweb_test.tests.base_test_case import TestBasic @test(groups=["cic_maintenance_mode"]) class CICMaintenanceMode(TestBasic): - """CICMaintenanceMode.""" # TODO documentation + """CICMaintenanceMode.""" @test(depends_on=[SetupEnvironment.prepare_slaves_5], groups=["cic_maintenance_mode_env"]) @@ -110,52 +106,56 @@ class CICMaintenanceMode(TestBasic): for devops_node in d_ctrls: _ip = self.fuel_web.get_nailgun_node_by_name( devops_node.name)['ip'] - with self.fuel_web.get_ssh_for_node(devops_node.name) as remote: - assert_true('True' in check_available_mode(_ip), - "Maintenance mode is not available") + logger.info('Maintenance mode for node {0}' + .format(devops_node.name)) + asserts.assert_true('True' in checkers.check_available_mode(_ip), + "Maintenance mode is not available") + self.ssh_manager.execute_on_remote( + ip=_ip, + cmd="umm on") - logger.info('Maintenance mode for node %s', devops_node.name) - result = remote.execute('umm on') - assert_equal(result['exit_code'], 0, - 'Failed to execute "{0}" on remote host: {1}'. - format('umm on', result)) - logger.info('Wait a %s node offline status after switching ' - 'maintenance mode ', devops_node.name) + logger.info('Wait a {0} node offline status after switching ' + 'maintenance mode '.format(devops_node.name)) try: wait( lambda: not self.fuel_web.get_nailgun_node_by_devops_node(devops_node) ['online'], timeout=60 * 10) except TimeoutError: - assert_false( + asserts.assert_false( self.fuel_web.get_nailgun_node_by_devops_node(devops_node) ['online'], 'Node {0} has not become offline after' 'switching maintenance mode'.format(devops_node.name)) - logger.info('Check that %s node in maintenance mode after ' - 'switching', devops_node.name) + logger.info('Check that {0} node in maintenance mode after ' + 'switching'.format(devops_node.name)) _ip = self.fuel_web.get_nailgun_node_by_name( devops_node.name)['ip'] - wait(lambda: tcp_ping(_ip, 22), timeout=60 * 10) - with self.fuel_web.get_ssh_for_node(devops_node.name) as remote: - assert_true('True' in check_auto_mode(_ip), - "Maintenance mode is not switch") + asserts.assert_true( + checkers.check_ping(self.env.get_admin_node_ip(), + _ip, + deadline=600), + "Host {0} is not reachable by ping during 600 sec" + .format(_ip)) - result = remote.execute('umm off') - assert_equal(result['exit_code'], 0, - 'Failed to execute "{0}" on remote host: {1}'. - format('umm off', result)) + asserts.assert_true('True' in checkers.check_auto_mode(_ip), + "Maintenance mode is not switch") - logger.info('Wait a %s node online status', devops_node.name) + self.ssh_manager.execute_on_remote( + ip=_ip, + cmd="umm off") + + logger.info('Wait a {0} node online status' + .format(devops_node.name)) try: wait( lambda: self.fuel_web.get_nailgun_node_by_devops_node(devops_node) ['online'], timeout=60 * 10) except TimeoutError: - assert_true( + asserts.assert_true( self.fuel_web.get_nailgun_node_by_devops_node(devops_node) ['online'], 'Node {0} has not become online after ' @@ -221,70 +221,83 @@ class CICMaintenanceMode(TestBasic): for devops_node in d_ctrls: _ip = self.fuel_web.get_nailgun_node_by_name( devops_node.name)['ip'] - with self.fuel_web.get_ssh_for_node(devops_node.name) as remote: - assert_true('True' in check_available_mode(_ip), - "Maintenance mode is not available") - logger.info('Change UMM.CONF on node %s', devops_node.name) - command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n" - "COUNTER_RESET_TIME=10' > /etc/umm.conf") + asserts.assert_true('True' in checkers.check_available_mode(_ip), + "Maintenance mode is not available") - result = remote.execute(command1) - assert_equal(result['exit_code'], 0, - 'Failed to execute "{0}" on remote host: {1}'. - format(command1, result)) + command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n" + "COUNTER_RESET_TIME=10' > /etc/umm.conf") - logger.info('Unexpected reboot on node %s', devops_node.name) - command2 = 'reboot --force >/dev/null & ' - result = remote.execute(command2) - assert_equal(result['exit_code'], 0, - 'Failed to execute "{0}" on remote host: {1}'. - format(command2, result)) - wait(lambda: not tcp_ping(_ip, 22), timeout=60 * 10) + self.ssh_manager.execute_on_remote( + ip=_ip, + cmd=command1) - logger.info('Wait a %s node offline status after unexpected ' - 'reboot', devops_node.name) + logger.info('Change UMM.CONF on node {0}' + .format(devops_node.name)) + + logger.info('Unexpected reboot on node {0}' + .format(devops_node.name)) + + command2 = 'reboot --force >/dev/null & ' + + self.ssh_manager.execute_on_remote( + ip=_ip, + cmd=command2) + + wait(lambda: + not checkers.check_ping(self.env.get_admin_node_ip(), + _ip), + timeout=60 * 10) + + logger.info('Wait a {0} node offline status after unexpected ' + 'reboot'.format(devops_node.name)) try: wait( lambda: not self.fuel_web.get_nailgun_node_by_devops_node(devops_node) ['online'], timeout=60 * 10) except TimeoutError: - assert_false( + asserts.assert_false( self.fuel_web.get_nailgun_node_by_devops_node(devops_node) ['online'], 'Node {0} has not become offline after unexpected' 'reboot'.format(devops_node.name)) - logger.info('Check that %s node in maintenance mode after' - ' unexpected reboot', devops_node.name) + logger.info('Check that {0} node in maintenance mode after' + ' unexpected reboot'.format(devops_node.name)) - wait(lambda: tcp_ping(_ip, 22), timeout=60 * 10) - with self.fuel_web.get_ssh_for_node(devops_node.name) as remote: - assert_true('True' in check_auto_mode(_ip), - "Maintenance mode is not switch") + asserts.assert_true( + checkers.check_ping(self.env.get_admin_node_ip(), + _ip, + deadline=600), + "Host {0} is not reachable by ping during 600 sec" + .format(_ip)) - result = remote.execute('umm off') - assert_equal(result['exit_code'], 0, - 'Failed to execute "{0}" on remote host: {1}'. - format('umm off', result)) - # Wait umm stops - time.sleep(30) - command3 = ("echo -e 'UMM=yes\nREBOOT_COUNT=2\n" - "COUNTER_RESET_TIME=10' > /etc/umm.conf") - result = remote.execute(command3) - assert_equal(result['exit_code'], 0, - 'Failed to execute "{0}" on remote host: {1}'. - format(command3, result)) + asserts.assert_true('True' in checkers.check_auto_mode(_ip), + "Maintenance mode is not switch") - logger.info('Wait a %s node online status', devops_node.name) + self.ssh_manager.execute_on_remote( + ip=_ip, + cmd="umm off") + + time.sleep(30) + + command3 = ("echo -e 'UMM=yes\nREBOOT_COUNT=2\n" + "COUNTER_RESET_TIME=10' > /etc/umm.conf") + + self.ssh_manager.execute_on_remote( + ip=_ip, + cmd=command3) + + logger.info('Wait a {0} node online status' + .format(devops_node.name)) try: wait( lambda: self.fuel_web.get_nailgun_node_by_devops_node(devops_node) ['online'], timeout=90 * 10) except TimeoutError: - assert_true( + asserts.assert_true( self.fuel_web.get_nailgun_node_by_devops_node(devops_node) ['online'], 'Node {0} has not become online after umm off'.format( @@ -350,31 +363,27 @@ class CICMaintenanceMode(TestBasic): for devops_node in d_ctrls: _ip = self.fuel_web.get_nailgun_node_by_name( devops_node.name)['ip'] - with self.fuel_web.get_ssh_for_node(devops_node.name) as remote: - assert_true('True' in check_available_mode(_ip), - "Maintenance mode is not available") + asserts.assert_true('True' in checkers.check_available_mode(_ip), + "Maintenance mode is not available") + self.ssh_manager.execute_on_remote( + ip=_ip, + cmd="umm disable") - logger.info('Maintenance mode for node %s is disable', - devops_node.name) - result = remote.execute('umm disable') - assert_equal(result['exit_code'], 0, - 'Failed to execute "{0}" on remote host: {1}'. - format('umm disable', result)) + asserts.assert_false('True' in checkers.check_available_mode(_ip), + "Maintenance mode should not be available") - assert_false('True' in check_available_mode(_ip), - "Maintenance mode should not be available") + logger.info('Try to execute maintenance mode ' + 'for node {0}'.format(devops_node.name)) - logger.info('Try to execute maintenance mode for node %s', - devops_node.name) - result = remote.execute('umm on') - assert_equal(result['exit_code'], 1, - 'Failed to execute "{0}" on remote host: {1}'. - format('umm on', result)) + self.ssh_manager.execute_on_remote( + ip=_ip, + cmd="umm on", + assert_ec_equal=[1]) # If we don't disable maintenance mode, # the node would have gone to reboot, so we just expect time.sleep(30) - assert_true( + asserts.assert_true( self.fuel_web.get_nailgun_node_by_devops_node(devops_node) ['online'], 'Node {0} should be online after command "umm on"'. @@ -420,50 +429,57 @@ class CICMaintenanceMode(TestBasic): for devops_node in d_ctrls: _ip = self.fuel_web.get_nailgun_node_by_name( devops_node.name)['ip'] - with self.fuel_web.get_ssh_for_node(devops_node.name) as remote: - assert_true('True' in check_available_mode(_ip), - "Maintenance mode is not available") + asserts.assert_true('True' in checkers.check_available_mode(_ip), + "Maintenance mode is not available") + logger.info('Change UMM.CONF on node {0}'.format(devops_node.name)) - logger.info('Change UMM.CONF on node %s', devops_node.name) - command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n" - "COUNTER_RESET_TIME=10' > /etc/umm.conf") + command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n" + "COUNTER_RESET_TIME=10' > /etc/umm.conf") + self.ssh_manager.execute_on_remote( + ip=_ip, + cmd=command1) - result = remote.execute(command1) - assert_equal(result['exit_code'], 0, - 'Failed to execute "{0}" on remote host: {1}'. - format(command1, result)) + self.ssh_manager.execute_on_remote( + ip=_ip, + cmd="umm disable") - result = remote.execute('umm disable') - assert_equal(result['exit_code'], 0, - 'Failed to execute "{0}" on remote host: {1}'. - format('umm disable', result)) + asserts.assert_false('True' in checkers.check_available_mode(_ip), + "Maintenance mode should not be available") - assert_false('True' in check_available_mode(_ip), - "Maintenance mode should not be available") + command2 = 'reboot --force >/dev/null & ' - logger.info('Unexpected reboot on node %s', devops_node.name) - command2 = 'reboot --force >/dev/null & ' - result = remote.execute(command2) - assert_equal(result['exit_code'], 0, - 'Failed to execute "{0}" on remote host: {1}'. - format(command2, result)) - wait(lambda: not tcp_ping(_ip, 22), timeout=60 * 10) + logger.info('Unexpected reboot on node {0}' + .format(devops_node.name)) + + self.ssh_manager.execute_on_remote( + ip=_ip, + cmd=command2) + + wait(lambda: + not checkers.check_ping(self.env.get_admin_node_ip(), + _ip), + timeout=60 * 10) # Node don't have enough time for set offline status # after reboot --force # Just waiting - wait(lambda: tcp_ping(_ip, 22), timeout=60 * 10) + asserts.assert_true( + checkers.check_ping(self.env.get_admin_node_ip(), + _ip, + deadline=600), + "Host {0} is not reachable by ping during 600 sec" + .format(_ip)) + logger.info('Wait a {0} node online status after unexpected ' + 'reboot'.format(devops_node.name)) - logger.info('Wait a %s node online status after unexpected ' - 'reboot', devops_node.name) self.fuel_web.wait_nodes_get_online_state([devops_node]) - logger.info('Check that %s node not in maintenance mode after' - ' unexpected reboot', devops_node.name) + logger.info('Check that {0} node not in maintenance mode after' + ' unexpected reboot'.format(devops_node.name)) - assert_false('True' in check_auto_mode(_ip), - "Maintenance mode should not switched") + asserts.assert_false('True' in checkers.check_auto_mode(_ip), + "Maintenance mode should not switched") # Wait until MySQL Galera is UP on some controller self.fuel_web.wait_mysql_galera_is_up(