Merge "Use the ssh_manager in cic_mm mode tests" into stable/mitaka
This commit is contained in:
Jenkins
Gerrit Code Review
commit
9432d9b701
@ -15,15 +15,11 @@ import time
|
||||
|
||||
from devops.error import TimeoutError
|
||||
from devops.helpers.helpers import _wait
|
||||
from devops.helpers.helpers import tcp_ping
|
||||
from devops.helpers.helpers import wait
|
||||
from proboscis.asserts import assert_equal
|
||||
from proboscis.asserts import assert_false
|
||||
from proboscis.asserts import assert_true
|
||||
from proboscis import asserts
|
||||
from proboscis import test
|
||||
|
||||
from fuelweb_test.helpers.checkers import check_auto_mode
|
||||
from fuelweb_test.helpers.checkers import check_available_mode
|
||||
from fuelweb_test.helpers import checkers
|
||||
from fuelweb_test.helpers.decorators import log_snapshot_after_test
|
||||
from fuelweb_test import logger
|
||||
from fuelweb_test import ostf_test_mapping
|
||||
@ -34,7 +30,7 @@ from fuelweb_test.tests.base_test_case import TestBasic
|
||||
|
||||
@test(groups=["cic_maintenance_mode"])
|
||||
class CICMaintenanceMode(TestBasic):
|
||||
"""CICMaintenanceMode.""" # TODO documentation
|
||||
"""CICMaintenanceMode."""
|
||||
|
||||
@test(depends_on=[SetupEnvironment.prepare_slaves_5],
|
||||
groups=["cic_maintenance_mode_env"])
|
||||
@ -110,52 +106,56 @@ class CICMaintenanceMode(TestBasic):
|
||||
for devops_node in d_ctrls:
|
||||
_ip = self.fuel_web.get_nailgun_node_by_name(
|
||||
devops_node.name)['ip']
|
||||
with self.fuel_web.get_ssh_for_node(devops_node.name) as remote:
|
||||
assert_true('True' in check_available_mode(_ip),
|
||||
"Maintenance mode is not available")
|
||||
logger.info('Maintenance mode for node {0}'
|
||||
.format(devops_node.name))
|
||||
asserts.assert_true('True' in checkers.check_available_mode(_ip),
|
||||
"Maintenance mode is not available")
|
||||
self.ssh_manager.execute_on_remote(
|
||||
ip=_ip,
|
||||
cmd="umm on")
|
||||
|
||||
logger.info('Maintenance mode for node %s', devops_node.name)
|
||||
result = remote.execute('umm on')
|
||||
assert_equal(result['exit_code'], 0,
|
||||
'Failed to execute "{0}" on remote host: {1}'.
|
||||
format('umm on', result))
|
||||
logger.info('Wait a %s node offline status after switching '
|
||||
'maintenance mode ', devops_node.name)
|
||||
logger.info('Wait a {0} node offline status after switching '
|
||||
'maintenance mode '.format(devops_node.name))
|
||||
try:
|
||||
wait(
|
||||
lambda: not
|
||||
self.fuel_web.get_nailgun_node_by_devops_node(devops_node)
|
||||
['online'], timeout=60 * 10)
|
||||
except TimeoutError:
|
||||
assert_false(
|
||||
asserts.assert_false(
|
||||
self.fuel_web.get_nailgun_node_by_devops_node(devops_node)
|
||||
['online'],
|
||||
'Node {0} has not become offline after'
|
||||
'switching maintenance mode'.format(devops_node.name))
|
||||
|
||||
logger.info('Check that %s node in maintenance mode after '
|
||||
'switching', devops_node.name)
|
||||
logger.info('Check that {0} node in maintenance mode after '
|
||||
'switching'.format(devops_node.name))
|
||||
|
||||
_ip = self.fuel_web.get_nailgun_node_by_name(
|
||||
devops_node.name)['ip']
|
||||
wait(lambda: tcp_ping(_ip, 22), timeout=60 * 10)
|
||||
with self.fuel_web.get_ssh_for_node(devops_node.name) as remote:
|
||||
assert_true('True' in check_auto_mode(_ip),
|
||||
"Maintenance mode is not switch")
|
||||
asserts.assert_true(
|
||||
checkers.check_ping(self.env.get_admin_node_ip(),
|
||||
_ip,
|
||||
deadline=600),
|
||||
"Host {0} is not reachable by ping during 600 sec"
|
||||
.format(_ip))
|
||||
|
||||
result = remote.execute('umm off')
|
||||
assert_equal(result['exit_code'], 0,
|
||||
'Failed to execute "{0}" on remote host: {1}'.
|
||||
format('umm off', result))
|
||||
asserts.assert_true('True' in checkers.check_auto_mode(_ip),
|
||||
"Maintenance mode is not switch")
|
||||
|
||||
logger.info('Wait a %s node online status', devops_node.name)
|
||||
self.ssh_manager.execute_on_remote(
|
||||
ip=_ip,
|
||||
cmd="umm off")
|
||||
|
||||
logger.info('Wait a {0} node online status'
|
||||
.format(devops_node.name))
|
||||
try:
|
||||
wait(
|
||||
lambda:
|
||||
self.fuel_web.get_nailgun_node_by_devops_node(devops_node)
|
||||
['online'], timeout=60 * 10)
|
||||
except TimeoutError:
|
||||
assert_true(
|
||||
asserts.assert_true(
|
||||
self.fuel_web.get_nailgun_node_by_devops_node(devops_node)
|
||||
['online'],
|
||||
'Node {0} has not become online after '
|
||||
@ -221,70 +221,83 @@ class CICMaintenanceMode(TestBasic):
|
||||
for devops_node in d_ctrls:
|
||||
_ip = self.fuel_web.get_nailgun_node_by_name(
|
||||
devops_node.name)['ip']
|
||||
with self.fuel_web.get_ssh_for_node(devops_node.name) as remote:
|
||||
assert_true('True' in check_available_mode(_ip),
|
||||
"Maintenance mode is not available")
|
||||
|
||||
logger.info('Change UMM.CONF on node %s', devops_node.name)
|
||||
command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n"
|
||||
"COUNTER_RESET_TIME=10' > /etc/umm.conf")
|
||||
asserts.assert_true('True' in checkers.check_available_mode(_ip),
|
||||
"Maintenance mode is not available")
|
||||
|
||||
result = remote.execute(command1)
|
||||
assert_equal(result['exit_code'], 0,
|
||||
'Failed to execute "{0}" on remote host: {1}'.
|
||||
format(command1, result))
|
||||
command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n"
|
||||
"COUNTER_RESET_TIME=10' > /etc/umm.conf")
|
||||
|
||||
logger.info('Unexpected reboot on node %s', devops_node.name)
|
||||
command2 = 'reboot --force >/dev/null & '
|
||||
result = remote.execute(command2)
|
||||
assert_equal(result['exit_code'], 0,
|
||||
'Failed to execute "{0}" on remote host: {1}'.
|
||||
format(command2, result))
|
||||
wait(lambda: not tcp_ping(_ip, 22), timeout=60 * 10)
|
||||
self.ssh_manager.execute_on_remote(
|
||||
ip=_ip,
|
||||
cmd=command1)
|
||||
|
||||
logger.info('Wait a %s node offline status after unexpected '
|
||||
'reboot', devops_node.name)
|
||||
logger.info('Change UMM.CONF on node {0}'
|
||||
.format(devops_node.name))
|
||||
|
||||
logger.info('Unexpected reboot on node {0}'
|
||||
.format(devops_node.name))
|
||||
|
||||
command2 = 'reboot --force >/dev/null & '
|
||||
|
||||
self.ssh_manager.execute_on_remote(
|
||||
ip=_ip,
|
||||
cmd=command2)
|
||||
|
||||
wait(lambda:
|
||||
not checkers.check_ping(self.env.get_admin_node_ip(),
|
||||
_ip),
|
||||
timeout=60 * 10)
|
||||
|
||||
logger.info('Wait a {0} node offline status after unexpected '
|
||||
'reboot'.format(devops_node.name))
|
||||
try:
|
||||
wait(
|
||||
lambda: not
|
||||
self.fuel_web.get_nailgun_node_by_devops_node(devops_node)
|
||||
['online'], timeout=60 * 10)
|
||||
except TimeoutError:
|
||||
assert_false(
|
||||
asserts.assert_false(
|
||||
self.fuel_web.get_nailgun_node_by_devops_node(devops_node)
|
||||
['online'],
|
||||
'Node {0} has not become offline after unexpected'
|
||||
'reboot'.format(devops_node.name))
|
||||
|
||||
logger.info('Check that %s node in maintenance mode after'
|
||||
' unexpected reboot', devops_node.name)
|
||||
logger.info('Check that {0} node in maintenance mode after'
|
||||
' unexpected reboot'.format(devops_node.name))
|
||||
|
||||
wait(lambda: tcp_ping(_ip, 22), timeout=60 * 10)
|
||||
with self.fuel_web.get_ssh_for_node(devops_node.name) as remote:
|
||||
assert_true('True' in check_auto_mode(_ip),
|
||||
"Maintenance mode is not switch")
|
||||
asserts.assert_true(
|
||||
checkers.check_ping(self.env.get_admin_node_ip(),
|
||||
_ip,
|
||||
deadline=600),
|
||||
"Host {0} is not reachable by ping during 600 sec"
|
||||
.format(_ip))
|
||||
|
||||
result = remote.execute('umm off')
|
||||
assert_equal(result['exit_code'], 0,
|
||||
'Failed to execute "{0}" on remote host: {1}'.
|
||||
format('umm off', result))
|
||||
# Wait umm stops
|
||||
time.sleep(30)
|
||||
command3 = ("echo -e 'UMM=yes\nREBOOT_COUNT=2\n"
|
||||
"COUNTER_RESET_TIME=10' > /etc/umm.conf")
|
||||
result = remote.execute(command3)
|
||||
assert_equal(result['exit_code'], 0,
|
||||
'Failed to execute "{0}" on remote host: {1}'.
|
||||
format(command3, result))
|
||||
asserts.assert_true('True' in checkers.check_auto_mode(_ip),
|
||||
"Maintenance mode is not switch")
|
||||
|
||||
logger.info('Wait a %s node online status', devops_node.name)
|
||||
self.ssh_manager.execute_on_remote(
|
||||
ip=_ip,
|
||||
cmd="umm off")
|
||||
|
||||
time.sleep(30)
|
||||
|
||||
command3 = ("echo -e 'UMM=yes\nREBOOT_COUNT=2\n"
|
||||
"COUNTER_RESET_TIME=10' > /etc/umm.conf")
|
||||
|
||||
self.ssh_manager.execute_on_remote(
|
||||
ip=_ip,
|
||||
cmd=command3)
|
||||
|
||||
logger.info('Wait a {0} node online status'
|
||||
.format(devops_node.name))
|
||||
try:
|
||||
wait(
|
||||
lambda:
|
||||
self.fuel_web.get_nailgun_node_by_devops_node(devops_node)
|
||||
['online'], timeout=90 * 10)
|
||||
except TimeoutError:
|
||||
assert_true(
|
||||
asserts.assert_true(
|
||||
self.fuel_web.get_nailgun_node_by_devops_node(devops_node)
|
||||
['online'],
|
||||
'Node {0} has not become online after umm off'.format(
|
||||
@ -350,31 +363,27 @@ class CICMaintenanceMode(TestBasic):
|
||||
for devops_node in d_ctrls:
|
||||
_ip = self.fuel_web.get_nailgun_node_by_name(
|
||||
devops_node.name)['ip']
|
||||
with self.fuel_web.get_ssh_for_node(devops_node.name) as remote:
|
||||
assert_true('True' in check_available_mode(_ip),
|
||||
"Maintenance mode is not available")
|
||||
asserts.assert_true('True' in checkers.check_available_mode(_ip),
|
||||
"Maintenance mode is not available")
|
||||
self.ssh_manager.execute_on_remote(
|
||||
ip=_ip,
|
||||
cmd="umm disable")
|
||||
|
||||
logger.info('Maintenance mode for node %s is disable',
|
||||
devops_node.name)
|
||||
result = remote.execute('umm disable')
|
||||
assert_equal(result['exit_code'], 0,
|
||||
'Failed to execute "{0}" on remote host: {1}'.
|
||||
format('umm disable', result))
|
||||
asserts.assert_false('True' in checkers.check_available_mode(_ip),
|
||||
"Maintenance mode should not be available")
|
||||
|
||||
assert_false('True' in check_available_mode(_ip),
|
||||
"Maintenance mode should not be available")
|
||||
logger.info('Try to execute maintenance mode '
|
||||
'for node {0}'.format(devops_node.name))
|
||||
|
||||
logger.info('Try to execute maintenance mode for node %s',
|
||||
devops_node.name)
|
||||
result = remote.execute('umm on')
|
||||
assert_equal(result['exit_code'], 1,
|
||||
'Failed to execute "{0}" on remote host: {1}'.
|
||||
format('umm on', result))
|
||||
self.ssh_manager.execute_on_remote(
|
||||
ip=_ip,
|
||||
cmd="umm on",
|
||||
assert_ec_equal=[1])
|
||||
|
||||
# If we don't disable maintenance mode,
|
||||
# the node would have gone to reboot, so we just expect
|
||||
time.sleep(30)
|
||||
assert_true(
|
||||
asserts.assert_true(
|
||||
self.fuel_web.get_nailgun_node_by_devops_node(devops_node)
|
||||
['online'],
|
||||
'Node {0} should be online after command "umm on"'.
|
||||
@ -420,50 +429,57 @@ class CICMaintenanceMode(TestBasic):
|
||||
for devops_node in d_ctrls:
|
||||
_ip = self.fuel_web.get_nailgun_node_by_name(
|
||||
devops_node.name)['ip']
|
||||
with self.fuel_web.get_ssh_for_node(devops_node.name) as remote:
|
||||
assert_true('True' in check_available_mode(_ip),
|
||||
"Maintenance mode is not available")
|
||||
asserts.assert_true('True' in checkers.check_available_mode(_ip),
|
||||
"Maintenance mode is not available")
|
||||
logger.info('Change UMM.CONF on node {0}'.format(devops_node.name))
|
||||
|
||||
logger.info('Change UMM.CONF on node %s', devops_node.name)
|
||||
command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n"
|
||||
"COUNTER_RESET_TIME=10' > /etc/umm.conf")
|
||||
command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n"
|
||||
"COUNTER_RESET_TIME=10' > /etc/umm.conf")
|
||||
self.ssh_manager.execute_on_remote(
|
||||
ip=_ip,
|
||||
cmd=command1)
|
||||
|
||||
result = remote.execute(command1)
|
||||
assert_equal(result['exit_code'], 0,
|
||||
'Failed to execute "{0}" on remote host: {1}'.
|
||||
format(command1, result))
|
||||
self.ssh_manager.execute_on_remote(
|
||||
ip=_ip,
|
||||
cmd="umm disable")
|
||||
|
||||
result = remote.execute('umm disable')
|
||||
assert_equal(result['exit_code'], 0,
|
||||
'Failed to execute "{0}" on remote host: {1}'.
|
||||
format('umm disable', result))
|
||||
asserts.assert_false('True' in checkers.check_available_mode(_ip),
|
||||
"Maintenance mode should not be available")
|
||||
|
||||
assert_false('True' in check_available_mode(_ip),
|
||||
"Maintenance mode should not be available")
|
||||
command2 = 'reboot --force >/dev/null & '
|
||||
|
||||
logger.info('Unexpected reboot on node %s', devops_node.name)
|
||||
command2 = 'reboot --force >/dev/null & '
|
||||
result = remote.execute(command2)
|
||||
assert_equal(result['exit_code'], 0,
|
||||
'Failed to execute "{0}" on remote host: {1}'.
|
||||
format(command2, result))
|
||||
wait(lambda: not tcp_ping(_ip, 22), timeout=60 * 10)
|
||||
logger.info('Unexpected reboot on node {0}'
|
||||
.format(devops_node.name))
|
||||
|
||||
self.ssh_manager.execute_on_remote(
|
||||
ip=_ip,
|
||||
cmd=command2)
|
||||
|
||||
wait(lambda:
|
||||
not checkers.check_ping(self.env.get_admin_node_ip(),
|
||||
_ip),
|
||||
timeout=60 * 10)
|
||||
|
||||
# Node don't have enough time for set offline status
|
||||
# after reboot --force
|
||||
# Just waiting
|
||||
|
||||
wait(lambda: tcp_ping(_ip, 22), timeout=60 * 10)
|
||||
asserts.assert_true(
|
||||
checkers.check_ping(self.env.get_admin_node_ip(),
|
||||
_ip,
|
||||
deadline=600),
|
||||
"Host {0} is not reachable by ping during 600 sec"
|
||||
.format(_ip))
|
||||
logger.info('Wait a {0} node online status after unexpected '
|
||||
'reboot'.format(devops_node.name))
|
||||
|
||||
logger.info('Wait a %s node online status after unexpected '
|
||||
'reboot', devops_node.name)
|
||||
self.fuel_web.wait_nodes_get_online_state([devops_node])
|
||||
|
||||
logger.info('Check that %s node not in maintenance mode after'
|
||||
' unexpected reboot', devops_node.name)
|
||||
logger.info('Check that {0} node not in maintenance mode after'
|
||||
' unexpected reboot'.format(devops_node.name))
|
||||
|
||||
assert_false('True' in check_auto_mode(_ip),
|
||||
"Maintenance mode should not switched")
|
||||
asserts.assert_false('True' in checkers.check_auto_mode(_ip),
|
||||
"Maintenance mode should not switched")
|
||||
|
||||
# Wait until MySQL Galera is UP on some controller
|
||||
self.fuel_web.wait_mysql_galera_is_up(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user