# Copyright 2016 Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import time from proboscis import test from proboscis.asserts import assert_true # pylint: disable=redefined-builtin from six.moves import xrange # pylint: enable=redefined-builtin from fuelweb_test import logger from fuelweb_test import ostf_test_mapping from fuelweb_test import settings from fuelweb_test.helpers.decorators import log_snapshot_after_test from fuelweb_test.helpers.decorators import setup_teardown from fuelweb_test.helpers.rally import RallyBenchmarkTest from fuelweb_test.helpers.utils import fill_space from fuelweb_test.tests.base_test_case import SetupEnvironment from fuelweb_test.tests.tests_strength.test_load_base import TestLoadBase @test(groups=["repetitive_restart"]) class RepetitiveRestart(TestLoadBase): """Test class for test group devoted to the repetitive cold restart of all nodes. Contains test case with cluster in HA mode with ceph and 100 times reboot procedure. """ @test(depends_on=[SetupEnvironment.prepare_slaves_5], groups=["ceph_partitions_repetitive_cold_restart"]) @log_snapshot_after_test @setup_teardown(setup=TestLoadBase.prepare_load_ceph_ha) def ceph_partitions_repetitive_cold_restart(self): """Ceph-osd partitions on 30% ~start rally~ repetitive cold restart Scenario: 1. Revert snapshot 'prepare_load_ceph_ha' 2. Wait until MySQL Galera is UP on some controller 3. Check Ceph status 4. Run ostf 5. Fill ceph partitions on all nodes up to 30% 6. Check Ceph status 7. Run RALLY 8. 100 times repetitive reboot: 9. Cold restart of all nodes 10. Wait for HA services ready 11. Wait until MySQL Galera is UP on some controller 12. Run ostf Duration 1700m Snapshot ceph_partitions_repetitive_cold_restart """ self.show_step(1, initialize=True) self.env.revert_snapshot("prepare_load_ceph_ha") self.show_step(2) primary_controller = self.fuel_web.get_nailgun_primary_node( self.env.d_env.nodes().slaves[0]) self.fuel_web.wait_mysql_galera_is_up([primary_controller.name]) cluster_id = self.fuel_web.get_last_created_cluster() self.show_step(3) self.fuel_web.check_ceph_status(cluster_id) self.show_step(4) self.fuel_web.run_ostf(cluster_id=cluster_id) self.show_step(5) ceph_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( cluster_id, ['ceph-osd']) for node in ceph_nodes: ip = node['ip'] file_dir = self.ssh_manager.execute_on_remote( ip=ip, cmd="mount | grep -m 1 ceph | awk '{printf($3)}'")['stdout'][0] fill_space(ip, file_dir, 30) self.show_step(6) self.fuel_web.check_ceph_status(cluster_id) self.show_step(7) assert_true(settings.PATCHING_RUN_RALLY, 'PATCHING_RUN_RALLY was not set in true') rally_benchmarks = {} benchmark_results = {} for tag in set(settings.RALLY_TAGS): rally_benchmarks[tag] = RallyBenchmarkTest( container_repo=settings.RALLY_DOCKER_REPO, environment=self.env, cluster_id=cluster_id, test_type=tag ) benchmark_results[tag] = rally_benchmarks[tag].run() logger.debug(benchmark_results[tag].show()) self.show_step(8) for i in xrange(settings.RESTART_COUNT): self.show_step(9, 'number {}'.format(i + 1), initialize=True) self.fuel_web.cold_restart_nodes( self.env.d_env.get_nodes(name__in=[ 'slave-01', 'slave-02', 'slave-03', 'slave-04', 'slave-05'])) self.show_step(10) self.fuel_web.assert_ha_services_ready(cluster_id) self.fuel_web.assert_os_services_ready(cluster_id) self.show_step(11) self.fuel_web.wait_mysql_galera_is_up([primary_controller.name]) try: self.fuel_web.run_single_ostf_test( cluster_id, test_sets=['smoke'], test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get( 'Create volume and attach it to instance')) except AssertionError: logger.debug("Test failed from first probe," " we sleep 180 seconds and try one more time " "and if it fails again - test will fail ") time.sleep(180) self.fuel_web.run_single_ostf_test( cluster_id, test_sets=['smoke'], test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get( 'Create volume and attach it to instance')) self.show_step(12) # LB 1519018 self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("ceph_partitions_repetitive_cold_restart")