diff --git a/tobiko/tests/faults/ha/test_cloud_recovery.py b/tobiko/tests/faults/ha/test_cloud_recovery.py index 6095e7b3f..7575c2bf9 100644 --- a/tobiko/tests/faults/ha/test_cloud_recovery.py +++ b/tobiko/tests/faults/ha/test_cloud_recovery.py @@ -1,7 +1,11 @@ from __future__ import absolute_import +import typing + +from oslo_log import log import testtools +import tobiko from tobiko.openstack import neutron from tobiko.openstack import tests from tobiko.tests.faults.ha import cloud_disruptions @@ -13,6 +17,9 @@ from tobiko.tripleo import undercloud from tobiko.tripleo import validations +LOG = log.getLogger(__name__) + + def overcloud_health_checks(passive_checks_only=False, skip_mac_table_size_test=True): # this method will be changed in future commit @@ -54,117 +61,157 @@ def check_overcloud_processes_health(): procs.ovn_overcloud_processes_validations) +class OvercloudHealthCheck(tobiko.SharedFixture): + + skips: typing.FrozenSet[str] = frozenset() + + @classmethod + def run_before(cls, **params): + cls.run(after=False, **params) + + @classmethod + def run_after(cls, **params): + cls.run(after=True, **params) + + @classmethod + def run(cls, after: bool, **params): + fixture = tobiko.get_fixture(cls) + params.setdefault('passive_checks_only', False) + params.setdefault('skip_mac_table_size_test', True) + skips = frozenset(k for k, v in params.items() if v) + if after or skips < fixture.skips: + # Force re-check + tobiko.cleanup_fixture(fixture) + else: + LOG.info("Will skip Overcloud health checks if already " + f"executed: {params}") + fixture.skips = skips + tobiko.setup_fixture(fixture) + + def setup_fixture(self): + # run validations + params = {name: True + for name in self.skips} + LOG.info(f"Start executing Overcloud health checks: {params}.") + overcloud_health_checks(**params) + LOG.info(f"Overcloud health checks successfully executed: {params}.") + + def cleanup_fixture(self): + self.skips = frozenset() + + @undercloud.skip_if_missing_undercloud class DisruptTripleoNodesTest(testtools.TestCase): - """ HA Tests: run health check -> disruptive action -> health check disruptive_action: a function that runs some - disruptive scenarion on a overcloud""" + disruptive scenario on a overcloud""" + def test_0vercloud_health_check(self): - overcloud_health_checks(skip_mac_table_size_test=False) + OvercloudHealthCheck.run_before(skip_mac_table_size_test=False) def test_hard_reboot_controllers_recovery(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.reset_all_controller_nodes() - overcloud_health_checks() + OvercloudHealthCheck.run_after() def test_soft_reboot_computes_recovery(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.reset_all_compute_nodes(hard_reset=False) # verify VM status is updated after reboot nova.wait_for_all_instances_status('SHUTOFF') # start all VM instance # otherwise sidecar containers will not run after computes reboot nova.start_all_instances() - overcloud_health_checks(passive_checks_only=True) + OvercloudHealthCheck.run_after(passive_checks_only=True) # TODO(eolivare): the following test is skipped due to rhbz#1890895 # def test_hard_reboot_computes_recovery(self): - # overcloud_health_checks() + # OvercloudHealthCheck.run_before() # cloud_disruptions.reset_all_compute_nodes(hard_reset=True) # # verify VM status is updated after reboot # nova.wait_for_all_instances_status('SHUTOFF') # # start all VM instance # # otherwise sidecar containers will not run after computes reboot # nova.start_all_instances() - # overcloud_health_checks(passive_checks_only=True) + # OvercloudHealthCheck.run_after(passive_checks_only=True) def test_reboot_controller_main_vip(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.reset_controller_main_vip() - overcloud_health_checks() + OvercloudHealthCheck.run_after() def test_reboot_controller_non_main_vip(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.reset_controllers_non_main_vip() - overcloud_health_checks() + OvercloudHealthCheck.run_after() def test_crash_controller_main_vip(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.crash_controller_main_vip() - overcloud_health_checks() + OvercloudHealthCheck.run_after() def test_crash_controller_non_main_vip(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.crash_controllers_non_main_vip() - overcloud_health_checks() + OvercloudHealthCheck.run_after() @pacemaker.skip_if_fencing_not_deployed def test_network_disruptor_main_vip(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.network_disrupt_controller_main_vip() - overcloud_health_checks() + OvercloudHealthCheck.run_after() cloud_disruptions.network_undisrupt_controller_main_vip() # @pacemaker.skip_if_fencing_not_deployed # def test_network_disruptor_non_main_vip(self): - # overcloud_health_checks() + # OvercloudHealthCheck.run_before() # cloud_disruptions.network_disrupt_controllers_non_main_vip() - # overcloud_health_checks() + # OvercloudHealthCheck.run_after() # cloud_disruptions.network_undisrupt_controllers_non_main_vip() @neutron.skip_unless_is_ovn() def test_reset_ovndb_pcs_master_resource(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.reset_ovndb_pcs_master_resource() - overcloud_health_checks() + OvercloudHealthCheck.run_after() @neutron.skip_unless_is_ovn() def test_reset_ovndb_pcs_resource(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.reset_ovndb_pcs_resource() - overcloud_health_checks() + OvercloudHealthCheck.run_after() @neutron.skip_unless_is_ovn() def test_reset_ovndb_master_container(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.reset_ovndb_master_container() - overcloud_health_checks() + OvercloudHealthCheck.run_after() def test_kill_rabbitmq_service_one_controller(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.kill_rabbitmq_service() - overcloud_health_checks() + OvercloudHealthCheck.run_after() def test_kill_all_galera_services(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.kill_all_galera_services() - overcloud_health_checks() + OvercloudHealthCheck.run_after() def test_remove_all_grastate_galera(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.remove_all_grastate_galera() - overcloud_health_checks() + OvercloudHealthCheck.run_after() def test_remove_one_grastate_galera(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.remove_one_grastate_galera() - overcloud_health_checks() + OvercloudHealthCheck.run_after() def test_request_galera_sst(self): - overcloud_health_checks() + OvercloudHealthCheck.run_before() cloud_disruptions.request_galera_sst() - overcloud_health_checks() + OvercloudHealthCheck.run_after() + # [..] # more tests to follow