diff --git a/doc/base_tests.rst b/doc/base_tests.rst index 4d0ff7cf7..198708172 100644 --- a/doc/base_tests.rst +++ b/doc/base_tests.rst @@ -388,6 +388,11 @@ Image based tests .. automodule:: fuelweb_test.tests.tests_strength.test_image_based :members: +Load tests +---------- +.. automodule:: fuelweb_test.tests.tests_strength.test_load + :members: + Master node failover tests -------------------------- .. automodule:: fuelweb_test.tests.tests_strength.test_master_node_failover diff --git a/fuelweb_test/helpers/rally.py b/fuelweb_test/helpers/rally.py index 316cacc11..e09c323f1 100644 --- a/fuelweb_test/helpers/rally.py +++ b/fuelweb_test/helpers/rally.py @@ -304,6 +304,15 @@ class RallyTask(object): "Rally task creation failed: {0}".format(result)) self.uuid = task_uuid + def abort(self, task_id): + logger.debug('Stop Rally task {0}'.format(task_id)) + cmd = 'rally task abort {0}'.format(task_id) + self.engine.run_container_command(cmd) + assert_true( + self.status in ('finished', 'aborted'), + "Rally task {0} was not aborted; current task status " + "is {1}".format(task_id, self.status)) + def get_results(self): if self.status == 'finished': cmd = 'rally task results {0}'.format(self.uuid) @@ -403,13 +412,15 @@ class RallyBenchmarkTest(object): ) self.current_task = None - def run(self, timeout=60 * 10): + def run(self, timeout=60 * 10, result=True): self.current_task = RallyTask(self.deployment, self.test_type) logger.info('Starting Rally benchmark test...') self.current_task.start() assert_equal(self.current_task.status, 'running', 'Rally task was started, but it is not running, status: ' '{0}'.format(self.current_task.status)) - wait(lambda: self.current_task.status == 'finished', timeout=timeout) - logger.info('Rally benchmark test is finished.') - return RallyResult(json_results=self.current_task.get_results()) + if result: + wait(lambda: self.current_task.status == 'finished', + timeout=timeout) + logger.info('Rally benchmark test is finished.') + return RallyResult(json_results=self.current_task.get_results()) diff --git a/fuelweb_test/run_tests.py b/fuelweb_test/run_tests.py index 8cda3ad5c..838821c59 100644 --- a/fuelweb_test/run_tests.py +++ b/fuelweb_test/run_tests.py @@ -139,6 +139,7 @@ def import_tests(): from system_test.tests.plugins.plugin_example import test_plugin_example # noqa from system_test.tests.plugins.plugin_example import test_plugin_example_v3 # noqa from gates_tests.tests import test_review_in_fuel_agent # noqa + from tests.tests_strength import test_load # noqa def run_tests(): diff --git a/fuelweb_test/tests/tests_strength/test_load.py b/fuelweb_test/tests/tests_strength/test_load.py new file mode 100644 index 000000000..dc6e2f8a4 --- /dev/null +++ b/fuelweb_test/tests/tests_strength/test_load.py @@ -0,0 +1,186 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import time + +from fuelweb_test.helpers.rally import RallyBenchmarkTest +from fuelweb_test.helpers.decorators import log_snapshot_after_test +from fuelweb_test import logger +from fuelweb_test import ostf_test_mapping as map_ostf +from fuelweb_test import settings +from fuelweb_test.tests.base_test_case import SetupEnvironment +from fuelweb_test.tests.base_test_case import TestBasic +from proboscis.asserts import assert_equal +from proboscis.asserts import assert_true +from proboscis import test + + +@test(groups=["load"]) +class Load(TestBasic): + """Load""" # TODO documentation + + @test(depends_on=[SetupEnvironment.prepare_slaves_5], + groups=["load_ceph_ha"]) + @log_snapshot_after_test + def load_ceph_ha(self): + """Prepare cluster in HA mode with ceph for load tests + + Scenario: + 1. Create cluster + 2. Add 3 nodes with controller + ceph-osd roles + 3. Add 2 node with compute role + 4. Deploy the cluster + 5. Make snapshot + + Duration 70m + Snapshot load_ceph_ha + """ + self.check_run("load_ceph_ha") + self.env.revert_snapshot("ready_with_5_slaves") + + self.show_step(1) + cluster_id = self.fuel_web.create_cluster( + name=self.__class__.__name__, + mode=settings.DEPLOYMENT_MODE, + settings={ + 'volumes_ceph': True, + 'images_ceph': True, + 'volumes_lvm': False, + 'osd_pool_size': "3" + } + ) + self.show_step(2) + self.show_step(3) + self.fuel_web.update_nodes( + cluster_id, + { + 'slave-01': ['controller', 'ceph-osd'], + 'slave-02': ['controller', 'ceph-osd'], + 'slave-03': ['controller', 'ceph-osd'], + 'slave-04': ['compute'], + 'slave-05': ['compute'] + } + ) + + self.show_step(4) + self.fuel_web.deploy_cluster_wait(cluster_id) + + self.fuel_web.verify_network(cluster_id) + self.fuel_web.run_ostf( + cluster_id=cluster_id) + self.show_step(5) + self.env.make_snapshot("load_ceph_ha", is_make=True) + + @test(depends_on=[load_ceph_ha], + groups=["load_ceph_partitions_cold_reboot"]) + @log_snapshot_after_test + def load_ceph_partitions_cold_reboot(self): + """Load ceph-osd partitions on 30% ~start rally~ reboot nodes + + Scenario: + 1. Revert snapshot 'load_ceph_ha' + 2. Wait until MySQL Galera is UP on some controller + 3. Check Ceph status + 4. Run ostf + 5. Fill ceph partitions on all nodes up to 30% + 6. Check Ceph status + 7. Run RALLY + 8. Cold restart all nodes + 9. Wait for HA services ready + 10. Wait until MySQL Galera is UP on some controller + 11. Run ostf + + Duration 30m + """ + self.show_step(1) + self.env.revert_snapshot("load_ceph_ha") + + self.show_step(2) + self.fuel_web.wait_mysql_galera_is_up(['slave-01']) + cluster_id = self.fuel_web.get_last_created_cluster() + + self.show_step(3) + self.fuel_web.check_ceph_status(cluster_id) + + self.show_step(4) + self.fuel_web.run_ostf(cluster_id=cluster_id) + + self.show_step(5) + for node in ['slave-0{0}'.format(slave) for slave in xrange(1, 4)]: + with self.fuel_web.get_ssh_for_node(node) as remote: + file_name = "test_data" + file_dir = remote.execute( + 'mount | grep -m 1 ceph')['stdout'][0].split()[2] + file_path = os.path.join(file_dir, file_name) + result = remote.execute( + 'fallocate -l 30G {0}'.format(file_path))['exit_code'] + assert_equal(result, 0, "The file {0} was not " + "allocated".format(file_name)) + + self.show_step(6) + self.fuel_web.check_ceph_status(cluster_id) + + self.show_step(7) + assert_true(settings.PATCHING_RUN_RALLY, + 'PATCHING_RUN_RALLY was not set in true') + rally_benchmarks = {} + for tag in set(settings.RALLY_TAGS): + rally_benchmarks[tag] = RallyBenchmarkTest( + container_repo=settings.RALLY_DOCKER_REPO, + environment=self.env, + cluster_id=cluster_id, + test_type=tag + ) + rally_benchmarks[tag].run(result=False) + + self.show_step(8) + self.fuel_web.cold_restart_nodes( + self.env.d_env.get_nodes(name__in=[ + 'slave-01', + 'slave-02', + 'slave-03', + 'slave-04', + 'slave-05'])) + + for tag in rally_benchmarks: + task_id = rally_benchmarks[tag].current_task.uuid + rally_benchmarks[tag].current_task.abort(task_id) + + self.show_step(9) + self.fuel_web.assert_ha_services_ready(cluster_id) + + self.fuel_web.assert_os_services_ready(cluster_id) + + self.show_step(10) + self.fuel_web.wait_mysql_galera_is_up(['slave-01']) + + try: + self.fuel_web.run_single_ostf_test( + cluster_id, test_sets=['smoke'], + test_name=map_ostf.OSTF_TEST_MAPPING.get( + 'Create volume and attach it to instance')) + except AssertionError: + logger.debug("Test failed from first probe," + " we sleep 180 seconds and try one more time " + "and if it fails again - test will fail ") + time.sleep(180) + self.fuel_web.run_single_ostf_test( + cluster_id, test_sets=['smoke'], + test_name=map_ostf.OSTF_TEST_MAPPING.get( + 'Create volume and attach it to instance')) + self.show_step(11) + # LB 1519018 + self.fuel_web.run_ostf(cluster_id=cluster_id) + self.env.make_snapshot("load_ceph_partitions_cold_reboot")