openstack/fuel-qa
Code Issues Proposed changes

Merge "Create tests for controller maintenance mode"

Browse Source
This commit is contained in:
Jenkins 2015-04-06 14:40:06 +00:00 committed by Gerrit Code Review
commit f1aa8ad9f9
3 changed files with 307 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
fuelweb_test/helpers/checkers.py
View File

@ -1026,3 +1026,21 @@ def check_nova_dhcp_lease(remote, instance_ip, instance_mac, node_dhcp_ip):
res_str = ''.join(res['stdout'])
logger.debug("DHCP server answer: {}".format(res_str))
return ' ack ' in res_str
def check_available_mode(remote):
command = ('umm status | grep runlevel &>/dev/null && echo "True" '
'|| echo "False"')
if remote.execute(command)['exit_code'] == 0:
return ''.join(remote.execute(command)['stdout']).strip()
else:
return ''.join(remote.execute(command)['stderr']).strip()
def check_auto_mode(remote):
command = ('umm status | grep umm &>/dev/null && echo "True" '
'|| echo "False"')
if remote.execute(command)['exit_code'] == 0:
return ''.join(remote.execute(command)['stdout']).strip()
else:
return ''.join(remote.execute(command)['stderr']).strip()

1
fuelweb_test/run_tests.py
View File

@ -32,6 +32,7 @@ def import_tests():
from tests.tests_strength import test_restart # noqa
from tests.tests_strength import test_huge_environments # noqa
from tests.tests_strength import test_image_based # noqa
from tests.tests_strength import test_cic_maintenance_mode # noqa
from tests import test_bonding # noqa
from tests.tests_strength import test_neutron # noqa
from tests import test_zabbix # noqa

288
fuelweb_test/tests/tests_strength/test_cic_maintenance_mode.py Normal file
View File

@ -0,0 +1,288 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
from devops.error import TimeoutError
from devops.helpers.helpers import wait
from proboscis.asserts import assert_equal
from proboscis.asserts import assert_false
from proboscis.asserts import assert_true
from proboscis import test
from fuelweb_test.helpers.checkers import check_auto_mode
from fuelweb_test.helpers.checkers import check_available_mode
from fuelweb_test.helpers.decorators import log_snapshot_on_error
from fuelweb_test import logger
from fuelweb_test import settings
from fuelweb_test.tests.base_test_case import SetupEnvironment
from fuelweb_test.tests.base_test_case import TestBasic
@test(groups=["cic_maintenance_mode"])
class CICMaintenanceMode(TestBasic):
@test(depends_on=[SetupEnvironment.prepare_slaves_5],
groups=["cic_maintenance_mode_env"])
@log_snapshot_on_error
def cic_maintenance_mode_env(self):
"""Deploy cluster in HA mode with 3 controller for maintenance mode
Scenario:
1. Create cluster
2. Add 3 node with controller and mongo roles
3. Add 2 node with compute and cinder roles
4. Deploy the cluster
Duration 100m
"""
self.env.revert_snapshot("ready_with_5_slaves")
data = {
'ceilometer': True
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE_HA,
settings=data)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['controller', 'mongo'],
'slave-02': ['controller', 'mongo'],
'slave-03': ['controller', 'mongo'],
'slave-04': ['compute', 'cinder'],
'slave-05': ['compute', 'cinder']
}
)
# Cluster deploy
self.fuel_web.deploy_cluster_wait(cluster_id)
# Check network
self.fuel_web.verify_network(cluster_id)
# Run ostf
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
self.env.make_snapshot("cic_maintenance_mode", is_make=True)
@test(depends_on=[cic_maintenance_mode_env],
groups=["manual_cic_maintenance_mode"])
@log_snapshot_on_error
def manual_cic_maintenance_mode(self):
"""Check manual maintenance mode for controller
Scenario:
1. Revert snapshot
2. Switch in maintenance mode
3. Wait until controller is rebooting
4. Exit maintenance mode
5. Check the controller become available
Duration 155m
"""
self.env.revert_snapshot('cic_maintenance_mode')
cluster_id = self.fuel_web.get_last_created_cluster()
for nailgun_node in self.env.d_env.nodes().slaves[0:3]:
remote = self.fuel_web.get_ssh_for_node(nailgun_node.name)
assert_true('True' in check_available_mode(remote),
"Maintenance mode is not available")
logger.info('Maintenance mode for node %s', nailgun_node.name)
result = remote.execute('umm on')
assert_equal(result['exit_code'], 0,
'Failed to execute "{0}" on remote host: {1}'.
format('umm on', result))
logger.info('Wait a %s node offline status after switching '
'maintenance mode ', nailgun_node.name)
try:
wait(
lambda: not
self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
['online'], timeout=60 * 10)
except TimeoutError:
assert_false(
self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
['online'],
'Node {0} has not become offline after'
'switching maintenance mode'.format(nailgun_node.name))
logger.info('Check that %s node in maintenance mode after '
'switching', nailgun_node.name)
remote = self.fuel_web.get_ssh_for_node(nailgun_node.name)
assert_true('True' in check_auto_mode(remote),
"Maintenance mode is not switch")
result = remote.execute('umm off')
assert_equal(result['exit_code'], 0,
'Failed to execute "{0}" on remote host: {1}'.
format('umm off', result))
logger.info('Wait a %s node online status', nailgun_node.name)
try:
wait(
lambda:
self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
['online'], timeout=60 * 10)
except TimeoutError:
assert_true(
self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
['online'],
'Node {0} has not become online after '
'exiting maintenance mode'.format(nailgun_node.name))
# Wait until MySQL Galera is UP on some controller
self.fuel_web.wait_mysql_galera_is_up(
[n.name for n in self.env.d_env.nodes().slaves[0:3]])
# Wait until Cinder services UP on a controller
self.fuel_web.wait_cinder_is_up(
[n.name for n in self.env.d_env.nodes().slaves[0:3]])
try:
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke',
'sanity'])
except AssertionError:
logger.debug("Test failed from first probe,"
" we sleep 1200 second try one more time"
" and if it fails again - test will fails ")
time.sleep(1200)
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke',
'sanity'])
@test(depends_on=[cic_maintenance_mode_env],
groups=["auto_cic_maintenance_mode"])
@log_snapshot_on_error
def auto_cic_maintenance_mode(self):
"""Check auto maintenance mode for controller
Scenario:
1. Revert snapshot
2. Unexpected reboot
3. Wait until controller is switching in maintenance mode
4. Exit maintenance mode
5. Check the controller become available
Duration 155m
"""
self.env.revert_snapshot('cic_maintenance_mode')
cluster_id = self.fuel_web.get_last_created_cluster()
for nailgun_node in self.env.d_env.nodes().slaves[0:3]:
remote = self.fuel_web.get_ssh_for_node(nailgun_node.name)
assert_true('True' in check_available_mode(remote),
"Maintenance mode is not available")
logger.info('Change UMM.CONF on node %s', nailgun_node.name)
command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n"
"COUNTER_RESET_TIME=10' > /etc/umm.conf")
result = remote.execute(command1)
assert_equal(result['exit_code'], 0,
'Failed to execute "{0}" on remote host: {1}'.
format(command1, result))
logger.info('Unexpected reboot on node %s', nailgun_node.name)
command2 = ('reboot --force >/dev/null & ')
result = remote.execute(command2)
assert_equal(result['exit_code'], 0,
'Failed to execute "{0}" on remote host: {1}'.
format(command2, result))
try:
wait(
lambda: not
self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
['online'], timeout=60 * 10)
except TimeoutError:
assert_false(
self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
['online'],
'Node {0} has not become offline after unexpected'
'reboot'.format(nailgun_node.name))
logger.info('Check that %s node in maintenance mode after'
' unexpected reboot', nailgun_node.name)
remote = self.fuel_web.get_ssh_for_node(nailgun_node.name)
assert_true('True' in check_auto_mode(remote),
"Maintenance mode is not switch")
result = remote.execute('umm off')
assert_equal(result['exit_code'], 0,
'Failed to execute "{0}" on remote host: {1}'.
format('umm off', result))
# Wait umm stops
time.sleep(30)
command3 = ("echo -e 'UMM=yes\nREBOOT_COUNT=2\n"
"COUNTER_RESET_TIME=10' > /etc/umm.conf")
result = remote.execute(command3)
assert_equal(result['exit_code'], 0,
'Failed to execute "{0}" on remote host: {1}'.
format(command3, result))
result = remote.execute('reboot')
assert_equal(result['exit_code'], 0,
'Failed to execute "{0}" on remote host: {1}'.
format('reboot', result))
logger.info('Wait a %s node offline status after reboot',
nailgun_node.name)
try:
wait(
lambda: not
self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
['online'], timeout=90 * 10)
except TimeoutError:
assert_false(
self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
['online'],
'Node {0} has not become offline after reboot'.format(
nailgun_node.name))
logger.info('Wait a %s node online status', nailgun_node.name)
try:
wait(
lambda:
self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
['online'], timeout=90 * 10)
except TimeoutError:
assert_true(
self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
['online'],
'Node {0} has not become online after reboot'.format(
nailgun_node.name))
# Wait until MySQL Galera is UP on some controller
self.fuel_web.wait_mysql_galera_is_up(
[n.name for n in self.env.d_env.nodes().slaves[0:3]])
# Wait until Cinder services UP on a controller
self.fuel_web.wait_cinder_is_up(
[n.name for n in self.env.d_env.nodes().slaves[0:3]])
try:
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke',
'sanity'])
except AssertionError:
logger.debug("Test failed from first probe,"
" we sleep 1200 second try one more time"
" and if it fails again - test will fails ")
time.sleep(1200)
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke',
'sanity'])