From 4f0c1c9a6f3375ba0b5b859002c678494cc1427d Mon Sep 17 00:00:00 2001 From: Veronica Krayneva Date: Mon, 23 Mar 2015 17:51:32 +0300 Subject: [PATCH] Create tests for controller maintenance mode Add first test for feature, which deploys cluster and switch in maintenance mode on each controller. The second test which deploys the cluster and automatically goes into the maintenance mode when unexpected reboot. Implements: blueprint controller-maintenance-mode-tests Change-Id: Ib94c584ff8de06dfa8f9d3fb7b4477a362b9a433 --- fuelweb_test/helpers/checkers.py | 18 ++ fuelweb_test/run_tests.py | 1 + .../test_cic_maintenance_mode.py | 288 ++++++++++++++++++ 3 files changed, 307 insertions(+) create mode 100644 fuelweb_test/tests/tests_strength/test_cic_maintenance_mode.py diff --git a/fuelweb_test/helpers/checkers.py b/fuelweb_test/helpers/checkers.py index dbb269cef..83e28a9cb 100644 --- a/fuelweb_test/helpers/checkers.py +++ b/fuelweb_test/helpers/checkers.py @@ -1013,3 +1013,21 @@ def check_nova_dhcp_lease(remote, instance_ip, instance_mac, node_dhcp_ip): res_str = ''.join(res['stdout']) logger.debug("DHCP server answer: {}".format(res_str)) return ' ack ' in res_str + + +def check_available_mode(remote): + command = ('umm status | grep runlevel &>/dev/null && echo "True" ' + '|| echo "False"') + if remote.execute(command)['exit_code'] == 0: + return ''.join(remote.execute(command)['stdout']).strip() + else: + return ''.join(remote.execute(command)['stderr']).strip() + + +def check_auto_mode(remote): + command = ('umm status | grep umm &>/dev/null && echo "True" ' + '|| echo "False"') + if remote.execute(command)['exit_code'] == 0: + return ''.join(remote.execute(command)['stdout']).strip() + else: + return ''.join(remote.execute(command)['stderr']).strip() diff --git a/fuelweb_test/run_tests.py b/fuelweb_test/run_tests.py index 727afe41c..2f2935623 100644 --- a/fuelweb_test/run_tests.py +++ b/fuelweb_test/run_tests.py @@ -32,6 +32,7 @@ def import_tests(): from tests.tests_strength import test_restart # noqa from tests.tests_strength import test_huge_environments # noqa from tests.tests_strength import test_image_based # noqa + from tests.tests_strength import test_cic_maintenance_mode # noqa from tests import test_bonding # noqa from tests.tests_strength import test_neutron # noqa from tests import test_zabbix # noqa diff --git a/fuelweb_test/tests/tests_strength/test_cic_maintenance_mode.py b/fuelweb_test/tests/tests_strength/test_cic_maintenance_mode.py new file mode 100644 index 000000000..63525da41 --- /dev/null +++ b/fuelweb_test/tests/tests_strength/test_cic_maintenance_mode.py @@ -0,0 +1,288 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import time + +from devops.error import TimeoutError +from devops.helpers.helpers import wait +from proboscis.asserts import assert_equal +from proboscis.asserts import assert_false +from proboscis.asserts import assert_true +from proboscis import test + +from fuelweb_test.helpers.checkers import check_auto_mode +from fuelweb_test.helpers.checkers import check_available_mode +from fuelweb_test.helpers.decorators import log_snapshot_on_error +from fuelweb_test import logger +from fuelweb_test import settings +from fuelweb_test.tests.base_test_case import SetupEnvironment +from fuelweb_test.tests.base_test_case import TestBasic + + +@test(groups=["cic_maintenance_mode"]) +class CICMaintenanceMode(TestBasic): + + @test(depends_on=[SetupEnvironment.prepare_slaves_5], + groups=["cic_maintenance_mode_env"]) + @log_snapshot_on_error + def cic_maintenance_mode_env(self): + """Deploy cluster in HA mode with 3 controller for maintenance mode + + Scenario: + 1. Create cluster + 2. Add 3 node with controller and mongo roles + 3. Add 2 node with compute and cinder roles + 4. Deploy the cluster + + Duration 100m + """ + self.env.revert_snapshot("ready_with_5_slaves") + data = { + 'ceilometer': True + } + + cluster_id = self.fuel_web.create_cluster( + name=self.__class__.__name__, + mode=settings.DEPLOYMENT_MODE_HA, + settings=data) + + self.fuel_web.update_nodes( + cluster_id, + { + 'slave-01': ['controller', 'mongo'], + 'slave-02': ['controller', 'mongo'], + 'slave-03': ['controller', 'mongo'], + 'slave-04': ['compute', 'cinder'], + 'slave-05': ['compute', 'cinder'] + } + ) + + # Cluster deploy + self.fuel_web.deploy_cluster_wait(cluster_id) + + # Check network + self.fuel_web.verify_network(cluster_id) + + # Run ostf + self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity']) + + self.env.make_snapshot("cic_maintenance_mode", is_make=True) + + @test(depends_on=[cic_maintenance_mode_env], + groups=["manual_cic_maintenance_mode"]) + @log_snapshot_on_error + def manual_cic_maintenance_mode(self): + """Check manual maintenance mode for controller + + Scenario: + 1. Revert snapshot + 2. Switch in maintenance mode + 3. Wait until controller is rebooting + 4. Exit maintenance mode + 5. Check the controller become available + + Duration 155m + """ + self.env.revert_snapshot('cic_maintenance_mode') + + cluster_id = self.fuel_web.get_last_created_cluster() + + for nailgun_node in self.env.d_env.nodes().slaves[0:3]: + remote = self.fuel_web.get_ssh_for_node(nailgun_node.name) + assert_true('True' in check_available_mode(remote), + "Maintenance mode is not available") + + logger.info('Maintenance mode for node %s', nailgun_node.name) + result = remote.execute('umm on') + assert_equal(result['exit_code'], 0, + 'Failed to execute "{0}" on remote host: {1}'. + format('umm on', result)) + logger.info('Wait a %s node offline status after switching ' + 'maintenance mode ', nailgun_node.name) + try: + wait( + lambda: not + self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node) + ['online'], timeout=60 * 10) + except TimeoutError: + assert_false( + self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node) + ['online'], + 'Node {0} has not become offline after' + 'switching maintenance mode'.format(nailgun_node.name)) + + logger.info('Check that %s node in maintenance mode after ' + 'switching', nailgun_node.name) + + remote = self.fuel_web.get_ssh_for_node(nailgun_node.name) + assert_true('True' in check_auto_mode(remote), + "Maintenance mode is not switch") + + result = remote.execute('umm off') + assert_equal(result['exit_code'], 0, + 'Failed to execute "{0}" on remote host: {1}'. + format('umm off', result)) + + logger.info('Wait a %s node online status', nailgun_node.name) + try: + wait( + lambda: + self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node) + ['online'], timeout=60 * 10) + except TimeoutError: + assert_true( + self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node) + ['online'], + 'Node {0} has not become online after ' + 'exiting maintenance mode'.format(nailgun_node.name)) + + # Wait until MySQL Galera is UP on some controller + self.fuel_web.wait_mysql_galera_is_up( + [n.name for n in self.env.d_env.nodes().slaves[0:3]]) + + # Wait until Cinder services UP on a controller + self.fuel_web.wait_cinder_is_up( + [n.name for n in self.env.d_env.nodes().slaves[0:3]]) + + try: + self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', + 'sanity']) + except AssertionError: + logger.debug("Test failed from first probe," + " we sleep 1200 second try one more time" + " and if it fails again - test will fails ") + time.sleep(1200) + self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', + 'sanity']) + + @test(depends_on=[cic_maintenance_mode_env], + groups=["auto_cic_maintenance_mode"]) + @log_snapshot_on_error + def auto_cic_maintenance_mode(self): + """Check auto maintenance mode for controller + + Scenario: + 1. Revert snapshot + 2. Unexpected reboot + 3. Wait until controller is switching in maintenance mode + 4. Exit maintenance mode + 5. Check the controller become available + + Duration 155m + """ + self.env.revert_snapshot('cic_maintenance_mode') + + cluster_id = self.fuel_web.get_last_created_cluster() + + for nailgun_node in self.env.d_env.nodes().slaves[0:3]: + remote = self.fuel_web.get_ssh_for_node(nailgun_node.name) + assert_true('True' in check_available_mode(remote), + "Maintenance mode is not available") + + logger.info('Change UMM.CONF on node %s', nailgun_node.name) + command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n" + "COUNTER_RESET_TIME=10' > /etc/umm.conf") + + result = remote.execute(command1) + assert_equal(result['exit_code'], 0, + 'Failed to execute "{0}" on remote host: {1}'. + format(command1, result)) + + logger.info('Unexpected reboot on node %s', nailgun_node.name) + command2 = ('reboot --force >/dev/null & ') + result = remote.execute(command2) + assert_equal(result['exit_code'], 0, + 'Failed to execute "{0}" on remote host: {1}'. + format(command2, result)) + + try: + wait( + lambda: not + self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node) + ['online'], timeout=60 * 10) + except TimeoutError: + assert_false( + self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node) + ['online'], + 'Node {0} has not become offline after unexpected' + 'reboot'.format(nailgun_node.name)) + + logger.info('Check that %s node in maintenance mode after' + ' unexpected reboot', nailgun_node.name) + + remote = self.fuel_web.get_ssh_for_node(nailgun_node.name) + assert_true('True' in check_auto_mode(remote), + "Maintenance mode is not switch") + + result = remote.execute('umm off') + assert_equal(result['exit_code'], 0, + 'Failed to execute "{0}" on remote host: {1}'. + format('umm off', result)) + # Wait umm stops + time.sleep(30) + command3 = ("echo -e 'UMM=yes\nREBOOT_COUNT=2\n" + "COUNTER_RESET_TIME=10' > /etc/umm.conf") + result = remote.execute(command3) + assert_equal(result['exit_code'], 0, + 'Failed to execute "{0}" on remote host: {1}'. + format(command3, result)) + + result = remote.execute('reboot') + assert_equal(result['exit_code'], 0, + 'Failed to execute "{0}" on remote host: {1}'. + format('reboot', result)) + logger.info('Wait a %s node offline status after reboot', + nailgun_node.name) + try: + wait( + lambda: not + self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node) + ['online'], timeout=90 * 10) + except TimeoutError: + assert_false( + self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node) + ['online'], + 'Node {0} has not become offline after reboot'.format( + nailgun_node.name)) + + logger.info('Wait a %s node online status', nailgun_node.name) + try: + wait( + lambda: + self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node) + ['online'], timeout=90 * 10) + except TimeoutError: + assert_true( + self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node) + ['online'], + 'Node {0} has not become online after reboot'.format( + nailgun_node.name)) + + # Wait until MySQL Galera is UP on some controller + self.fuel_web.wait_mysql_galera_is_up( + [n.name for n in self.env.d_env.nodes().slaves[0:3]]) + + # Wait until Cinder services UP on a controller + self.fuel_web.wait_cinder_is_up( + [n.name for n in self.env.d_env.nodes().slaves[0:3]]) + + try: + self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', + 'sanity']) + except AssertionError: + logger.debug("Test failed from first probe," + " we sleep 1200 second try one more time" + " and if it fails again - test will fails ") + time.sleep(1200) + self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', + 'sanity'])