From efb7e1b5145aa3ab61fbc3b3dc7367b25fbe6e7d Mon Sep 17 00:00:00 2001 From: Liubov Efremova Date: Thu, 14 Jan 2016 15:40:55 +0300 Subject: [PATCH] Repetitive power off/on test was added Change-Id: If207423cff94ec929d59dac27cdbe66bddbb7b58 Closes-Bug: #1523844 --- doc/base_tests.rst | 10 ++ fuelweb_test/helpers/utils.py | 17 +++ fuelweb_test/models/fuel_web_client.py | 32 ++-- .../tests/tests_strength/test_load.py | 116 +++++--------- .../tests/tests_strength/test_load_base.py | 77 ++++++++++ .../tests_strength/test_repetitive_restart.py | 144 ++++++++++++++++++ 6 files changed, 299 insertions(+), 97 deletions(-) create mode 100644 fuelweb_test/tests/tests_strength/test_load_base.py create mode 100644 fuelweb_test/tests/tests_strength/test_repetitive_restart.py diff --git a/doc/base_tests.rst b/doc/base_tests.rst index d1daf8073..74639aee6 100644 --- a/doc/base_tests.rst +++ b/doc/base_tests.rst @@ -558,6 +558,11 @@ Image based tests .. automodule:: fuelweb_test.tests.tests_strength.test_image_based :members: +Base load tests +--------------- +.. automodule:: fuelweb_test.tests.tests_strength.test_load_base + :members: + Load tests ---------- .. automodule:: fuelweb_test.tests.tests_strength.test_load @@ -583,6 +588,11 @@ OSTF repeatable tests .. automodule:: fuelweb_test.tests.tests_strength.test_ostf_repeatable_tests :members: +Repetitive restart tests +------------------------ +.. automodule:: fuelweb_test.tests.tests_strength.test_repetitive_restart + :members: + Restart tests ------------- .. automodule:: fuelweb_test.tests.tests_strength.test_restart diff --git a/fuelweb_test/helpers/utils.py b/fuelweb_test/helpers/utils.py index 488f456e8..6c7a9b312 100644 --- a/fuelweb_test/helpers/utils.py +++ b/fuelweb_test/helpers/utils.py @@ -888,3 +888,20 @@ def erase_data_from_hdd(remote, for cmd in commands: run_on_remote(remote, cmd) + + +@logwrap +def fill_space(ip, file_dir, size): + """Allocates space to some file in the specified directory + on the specified node + + :param ip: the ip of the node + :param file_dir: the specified directory + :param size: the amount of space in Gb + """ + file_name = "test_data" + file_path = os.path.join(file_dir, file_name) + SSHManager().execute_on_remote( + ip=ip, + cmd='fallocate -l {0}G {1}'.format(size, file_path), + err_msg="The file {0} was not allocated".format(file_name)) diff --git a/fuelweb_test/models/fuel_web_client.py b/fuelweb_test/models/fuel_web_client.py index b4ed28277..01a16fbbc 100644 --- a/fuelweb_test/models/fuel_web_client.py +++ b/fuelweb_test/models/fuel_web_client.py @@ -15,8 +15,6 @@ import re import time import traceback -import ipaddr -from netaddr import EUI from urllib2 import HTTPError from devops.error import DevopsCalledProcessError @@ -24,20 +22,24 @@ from devops.error import TimeoutError from devops.helpers.helpers import _wait from devops.helpers.helpers import wait from devops.models.node import Node -from fuelweb_test.helpers.ssh_manager import SSHManager -from fuelweb_test.helpers.ssl import copy_cert_from_master -from fuelweb_test.helpers.ssl import change_cluster_ssl_config +import ipaddr from ipaddr import IPNetwork +from netaddr import EUI from proboscis.asserts import assert_equal -from proboscis.asserts import assert_not_equal from proboscis.asserts import assert_false from proboscis.asserts import assert_is_not_none -from proboscis.asserts import assert_true +from proboscis.asserts import assert_not_equal from proboscis.asserts import assert_raises +from proboscis.asserts import assert_true import yaml +from fuelweb_test import logger +from fuelweb_test import logwrap +from fuelweb_test import ostf_test_mapping as map_ostf +from fuelweb_test import QuietLogger from fuelweb_test.helpers import ceph from fuelweb_test.helpers import checkers +from fuelweb_test.helpers import replace_repos from fuelweb_test.helpers.decorators import check_repos_management from fuelweb_test.helpers.decorators import custom_repo from fuelweb_test.helpers.decorators import download_astute_yaml @@ -46,23 +48,22 @@ from fuelweb_test.helpers.decorators import duration from fuelweb_test.helpers.decorators import retry from fuelweb_test.helpers.decorators import update_fuel from fuelweb_test.helpers.decorators import upload_manifests -from fuelweb_test.helpers import replace_repos from fuelweb_test.helpers.security import SecurityChecks -from fuelweb_test.helpers.utils import run_on_remote -from fuelweb_test.helpers.utils import node_freemem +from fuelweb_test.helpers.ssh_manager import SSHManager +from fuelweb_test.helpers.ssl import change_cluster_ssl_config +from fuelweb_test.helpers.ssl import copy_cert_from_master from fuelweb_test.helpers.utils import get_node_hiera_roles +from fuelweb_test.helpers.utils import node_freemem from fuelweb_test.helpers.utils import pretty_log -from fuelweb_test import logger -from fuelweb_test import logwrap +from fuelweb_test.helpers.utils import run_on_remote from fuelweb_test.models.nailgun_client import NailgunClient -from fuelweb_test import ostf_test_mapping as map_ostf -from fuelweb_test import QuietLogger import fuelweb_test.settings as help_data from fuelweb_test.settings import ATTEMPTS from fuelweb_test.settings import BONDING from fuelweb_test.settings import DEPLOYMENT_MODE_HA from fuelweb_test.settings import DISABLE_SSL from fuelweb_test.settings import DNS_SUFFIX +from fuelweb_test.settings import iface_alias from fuelweb_test.settings import KVM_USE from fuelweb_test.settings import MULTIPLE_NETWORKS from fuelweb_test.settings import NEUTRON @@ -77,13 +78,12 @@ from fuelweb_test.settings import REPLACE_DEFAULT_REPOS from fuelweb_test.settings import REPLACE_DEFAULT_REPOS_ONLY_ONCE from fuelweb_test.settings import SSL_CN from fuelweb_test.settings import TIMEOUT +from fuelweb_test.settings import USER_OWNED_CERT from fuelweb_test.settings import VCENTER_DATACENTER from fuelweb_test.settings import VCENTER_DATASTORE -from fuelweb_test.settings import USER_OWNED_CERT from fuelweb_test.settings import VCENTER_IP from fuelweb_test.settings import VCENTER_PASSWORD from fuelweb_test.settings import VCENTER_USERNAME -from fuelweb_test.settings import iface_alias class FuelWebClient(object): diff --git a/fuelweb_test/tests/tests_strength/test_load.py b/fuelweb_test/tests/tests_strength/test_load.py index 640912591..82e64bce3 100644 --- a/fuelweb_test/tests/tests_strength/test_load.py +++ b/fuelweb_test/tests/tests_strength/test_load.py @@ -12,85 +12,40 @@ # License for the specific language governing permissions and limitations # under the License. -import os import time -from fuelweb_test.helpers.rally import RallyBenchmarkTest -from fuelweb_test.helpers.decorators import log_snapshot_after_test +from proboscis import test +from proboscis.asserts import assert_true + from fuelweb_test import logger from fuelweb_test import ostf_test_mapping as map_ostf from fuelweb_test import settings +from fuelweb_test.helpers.decorators import log_snapshot_after_test +from fuelweb_test.helpers.decorators import setup_teardown +from fuelweb_test.helpers.rally import RallyBenchmarkTest +from fuelweb_test.helpers.utils import fill_space from fuelweb_test.tests.base_test_case import SetupEnvironment -from fuelweb_test.tests.base_test_case import TestBasic -from proboscis.asserts import assert_equal -from proboscis.asserts import assert_true -from proboscis import test +from fuelweb_test.tests.tests_strength.test_load_base import TestLoadBase @test(groups=["load"]) -class Load(TestBasic): - """Load""" # TODO documentation +class Load(TestLoadBase): + """Test class for the test group devoted to the load tests. + + Contains test case with cluster in HA mode with ceph, + launching Rally and cold restart of all nodes. + + """ @test(depends_on=[SetupEnvironment.prepare_slaves_5], - groups=["load_ceph_ha"]) - @log_snapshot_after_test - def load_ceph_ha(self): - """Prepare cluster in HA mode with ceph for load tests - - Scenario: - 1. Create cluster - 2. Add 3 nodes with controller + ceph-osd roles - 3. Add 2 node with compute role - 4. Deploy the cluster - 5. Make snapshot - - Duration 70m - Snapshot load_ceph_ha - """ - self.check_run("load_ceph_ha") - self.env.revert_snapshot("ready_with_5_slaves") - - self.show_step(1, initialize=True) - cluster_id = self.fuel_web.create_cluster( - name=self.__class__.__name__, - mode=settings.DEPLOYMENT_MODE, - settings={ - 'volumes_ceph': True, - 'images_ceph': True, - 'volumes_lvm': False, - 'osd_pool_size': "3" - } - ) - self.show_step(2) - self.show_step(3) - self.fuel_web.update_nodes( - cluster_id, - { - 'slave-01': ['controller', 'ceph-osd'], - 'slave-02': ['controller', 'ceph-osd'], - 'slave-03': ['controller', 'ceph-osd'], - 'slave-04': ['compute'], - 'slave-05': ['compute'] - } - ) - - self.show_step(4) - self.fuel_web.deploy_cluster_wait(cluster_id) - - self.fuel_web.verify_network(cluster_id) - self.fuel_web.run_ostf( - cluster_id=cluster_id) - self.show_step(5) - self.env.make_snapshot("load_ceph_ha", is_make=True) - - @test(depends_on=[load_ceph_ha], groups=["load_ceph_partitions_cold_reboot"]) @log_snapshot_after_test + @setup_teardown(setup=TestLoadBase.prepare_load_ceph_ha) def load_ceph_partitions_cold_reboot(self): """Load ceph-osd partitions on 30% ~start rally~ reboot nodes Scenario: - 1. Revert snapshot 'load_ceph_ha' + 1. Revert snapshot 'prepare_load_ceph_ha' 2. Wait until MySQL Galera is UP on some controller 3. Check Ceph status 4. Run ostf @@ -102,13 +57,16 @@ class Load(TestBasic): 10. Wait until MySQL Galera is UP on some controller 11. Run ostf - Duration 30m + Duration 180m + Snapshot load_ceph_partitions_cold_reboot """ self.show_step(1, initialize=True) - self.env.revert_snapshot("load_ceph_ha") + self.env.revert_snapshot("prepare_load_ceph_ha") self.show_step(2) - self.fuel_web.wait_mysql_galera_is_up(['slave-01']) + primary_controller = self.fuel_web.get_nailgun_primary_node( + self.env.d_env.nodes().slaves[0]) + self.fuel_web.wait_mysql_galera_is_up([primary_controller.name]) cluster_id = self.fuel_web.get_last_created_cluster() self.show_step(3) @@ -118,16 +76,14 @@ class Load(TestBasic): self.fuel_web.run_ostf(cluster_id=cluster_id) self.show_step(5) - for node in ['slave-0{0}'.format(slave) for slave in xrange(1, 4)]: - with self.fuel_web.get_ssh_for_node(node) as remote: - file_name = "test_data" - file_dir = remote.execute( - 'mount | grep -m 1 ceph')['stdout'][0].split()[2] - file_path = os.path.join(file_dir, file_name) - result = remote.execute( - 'fallocate -l 30G {0}'.format(file_path))['exit_code'] - assert_equal(result, 0, "The file {0} was not " - "allocated".format(file_name)) + ceph_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( + cluster_id, ['ceph-osd']) + for node in ceph_nodes: + ip = node['ip'] + file_dir = self.ssh_manager.execute_on_remote( + ip=ip, + cmd="mount | grep -m 1 ceph | awk '{printf($3)}'")['stdout'][0] + fill_space(ip, file_dir, 30) self.show_step(6) self.fuel_web.check_ceph_status(cluster_id) @@ -136,6 +92,7 @@ class Load(TestBasic): assert_true(settings.PATCHING_RUN_RALLY, 'PATCHING_RUN_RALLY was not set in true') rally_benchmarks = {} + benchmark_results = {} for tag in set(settings.RALLY_TAGS): rally_benchmarks[tag] = RallyBenchmarkTest( container_repo=settings.RALLY_DOCKER_REPO, @@ -143,7 +100,8 @@ class Load(TestBasic): cluster_id=cluster_id, test_type=tag ) - rally_benchmarks[tag].run(result=False) + benchmark_results[tag] = rally_benchmarks[tag].run() + logger.debug(benchmark_results[tag].show()) self.show_step(8) self.fuel_web.cold_restart_nodes( @@ -154,17 +112,13 @@ class Load(TestBasic): 'slave-04', 'slave-05'])) - for tag in rally_benchmarks: - task_id = rally_benchmarks[tag].current_task.uuid - rally_benchmarks[tag].current_task.abort(task_id) - self.show_step(9) self.fuel_web.assert_ha_services_ready(cluster_id) self.fuel_web.assert_os_services_ready(cluster_id) self.show_step(10) - self.fuel_web.wait_mysql_galera_is_up(['slave-01']) + self.fuel_web.wait_mysql_galera_is_up([primary_controller.name]) try: self.fuel_web.run_single_ostf_test( diff --git a/fuelweb_test/tests/tests_strength/test_load_base.py b/fuelweb_test/tests/tests_strength/test_load_base.py new file mode 100644 index 000000000..bfb3b0828 --- /dev/null +++ b/fuelweb_test/tests/tests_strength/test_load_base.py @@ -0,0 +1,77 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fuelweb_test.settings import DEPLOYMENT_MODE +from fuelweb_test.tests.base_test_case import TestBasic + + +class TestLoadBase(TestBasic): + """ + + This class contains basic methods for different load tests scenarios. + + """ + + def prepare_load_ceph_ha(self): + """Prepare cluster in HA mode with ceph for load tests + + Scenario: + 1. Create cluster + 2. Add 3 nodes with controller + ceph-osd roles + 3. Add 2 node with compute role + 4. Deploy the cluster + 5. Make snapshot + + Duration 70m + Snapshot prepare_load_ceph_ha + """ + + if self.env.d_env.has_snapshot("prepare_load_ceph_ha"): + return + + self.env.revert_snapshot("ready_with_5_slaves") + + self.show_step(1, initialize=True) + cluster_id = self.fuel_web.create_cluster( + name=self.__class__.__name__, + mode=DEPLOYMENT_MODE, + settings={ + 'volumes_ceph': True, + 'images_ceph': True, + 'volumes_lvm': False, + 'osd_pool_size': "3" + } + ) + self.show_step(2) + self.show_step(3) + self.fuel_web.update_nodes( + cluster_id, + { + 'slave-01': ['controller', 'ceph-osd'], + 'slave-02': ['controller', 'ceph-osd'], + 'slave-03': ['controller', 'ceph-osd'], + 'slave-04': ['compute'], + 'slave-05': ['compute'] + } + ) + + self.show_step(4) + self.fuel_web.deploy_cluster_wait(cluster_id) + + self.fuel_web.verify_network(cluster_id) + self.fuel_web.run_ostf( + cluster_id=cluster_id) + + self.show_step(5) + self.env.make_snapshot("prepare_load_ceph_ha", is_make=True) diff --git a/fuelweb_test/tests/tests_strength/test_repetitive_restart.py b/fuelweb_test/tests/tests_strength/test_repetitive_restart.py new file mode 100644 index 000000000..1da47291e --- /dev/null +++ b/fuelweb_test/tests/tests_strength/test_repetitive_restart.py @@ -0,0 +1,144 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time + +from proboscis import test +from proboscis.asserts import assert_true + +from fuelweb_test import logger +from fuelweb_test import ostf_test_mapping as map_ostf +from fuelweb_test import settings +from fuelweb_test.helpers.decorators import log_snapshot_after_test +from fuelweb_test.helpers.decorators import setup_teardown +from fuelweb_test.helpers.rally import RallyBenchmarkTest +from fuelweb_test.helpers.utils import fill_space +from fuelweb_test.tests.base_test_case import SetupEnvironment +from fuelweb_test.tests.tests_strength.test_load_base import TestLoadBase + + +@test(groups=["repetitive_restart"]) +class RepetitiveRestart(TestLoadBase): + """Test class for test group devoted to the repetitive cold restart + of all nodes. + + Contains test case with cluster in HA mode with ceph + and 100 times reboot procedure. + + """ + + @test(depends_on=[SetupEnvironment.prepare_slaves_5], + groups=["ceph_partitions_repetitive_cold_restart"]) + @log_snapshot_after_test + @setup_teardown(setup=TestLoadBase.prepare_load_ceph_ha) + def ceph_partitions_repetitive_cold_restart(self): + """Ceph-osd partitions on 30% ~start rally~ repetitive cold restart + + Scenario: + 1. Revert snapshot 'prepare_load_ceph_ha' + 2. Wait until MySQL Galera is UP on some controller + 3. Check Ceph status + 4. Run ostf + 5. Fill ceph partitions on all nodes up to 30% + 6. Check Ceph status + 7. Run RALLY + 8. 100 times repetitive reboot: + 9. Cold restart of all nodes + 10. Wait for HA services ready + 11. Wait until MySQL Galera is UP on some controller + 12. Run ostf + + Duration 1700m + Snapshot ceph_partitions_repetitive_cold_restart + """ + self.show_step(1, initialize=True) + self.env.revert_snapshot("prepare_load_ceph_ha") + + self.show_step(2) + primary_controller = self.fuel_web.get_nailgun_primary_node( + self.env.d_env.nodes().slaves[0]) + self.fuel_web.wait_mysql_galera_is_up([primary_controller.name]) + cluster_id = self.fuel_web.get_last_created_cluster() + + self.show_step(3) + self.fuel_web.check_ceph_status(cluster_id) + + self.show_step(4) + self.fuel_web.run_ostf(cluster_id=cluster_id) + + self.show_step(5) + ceph_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( + cluster_id, ['ceph-osd']) + for node in ceph_nodes: + ip = node['ip'] + file_dir = self.ssh_manager.execute_on_remote( + ip=ip, + cmd="mount | grep -m 1 ceph | awk '{printf($3)}'")['stdout'][0] + fill_space(ip, file_dir, 30) + + self.show_step(6) + self.fuel_web.check_ceph_status(cluster_id) + + self.show_step(7) + assert_true(settings.PATCHING_RUN_RALLY, + 'PATCHING_RUN_RALLY was not set in true') + rally_benchmarks = {} + benchmark_results = {} + for tag in set(settings.RALLY_TAGS): + rally_benchmarks[tag] = RallyBenchmarkTest( + container_repo=settings.RALLY_DOCKER_REPO, + environment=self.env, + cluster_id=cluster_id, + test_type=tag + ) + benchmark_results[tag] = rally_benchmarks[tag].run() + logger.debug(benchmark_results[tag].show()) + + self.show_step(8) + for i in xrange(100): + self.show_step(9, 'number {}'.format(i + 1), initialize=True) + self.fuel_web.cold_restart_nodes( + self.env.d_env.get_nodes(name__in=[ + 'slave-01', + 'slave-02', + 'slave-03', + 'slave-04', + 'slave-05'])) + + self.show_step(10) + self.fuel_web.assert_ha_services_ready(cluster_id) + + self.fuel_web.assert_os_services_ready(cluster_id) + + self.show_step(11) + self.fuel_web.wait_mysql_galera_is_up([primary_controller.name]) + + try: + self.fuel_web.run_single_ostf_test( + cluster_id, test_sets=['smoke'], + test_name=map_ostf.OSTF_TEST_MAPPING.get( + 'Create volume and attach it to instance')) + except AssertionError: + logger.debug("Test failed from first probe," + " we sleep 180 seconds and try one more time " + "and if it fails again - test will fail ") + time.sleep(180) + self.fuel_web.run_single_ostf_test( + cluster_id, test_sets=['smoke'], + test_name=map_ostf.OSTF_TEST_MAPPING.get( + 'Create volume and attach it to instance')) + self.show_step(12) + # LB 1519018 + self.fuel_web.run_ostf(cluster_id=cluster_id) + self.env.make_snapshot("ceph_partitions_repetitive_cold_restart")