Merge "Cloud recovery tests skip overcloud validations before disruption"

tobiko/tests/faults/ha/test_cloud_recovery.py

@@ -1,7 +1,11 @@
 from __future__ import absolute_import
 
+import typing
+
+from oslo_log import log
 import testtools
 
+import tobiko
 from tobiko.openstack import neutron
 from tobiko.openstack import tests
 from tobiko.tests.faults.ha import cloud_disruptions
@@ -13,6 +17,9 @@ from tobiko.tripleo import undercloud
 from tobiko.tripleo import validations
 
 
+LOG = log.getLogger(__name__)
+
+
 def overcloud_health_checks(passive_checks_only=False,
                             skip_mac_table_size_test=True):
     # this method will be changed in future commit
@@ -54,117 +61,157 @@ def check_overcloud_processes_health():
             procs.ovn_overcloud_processes_validations)
 
 
+class OvercloudHealthCheck(tobiko.SharedFixture):
+
+    skips: typing.FrozenSet[str] = frozenset()
+
+    @classmethod
+    def run_before(cls, **params):
+        cls.run(after=False, **params)
+
+    @classmethod
+    def run_after(cls, **params):
+        cls.run(after=True, **params)
+
+    @classmethod
+    def run(cls, after: bool, **params):
+        fixture = tobiko.get_fixture(cls)
+        params.setdefault('passive_checks_only', False)
+        params.setdefault('skip_mac_table_size_test', True)
+        skips = frozenset(k for k, v in params.items() if v)
+        if after or skips < fixture.skips:
+            # Force re-check
+            tobiko.cleanup_fixture(fixture)
+        else:
+            LOG.info("Will skip Overcloud health checks if already "
+                     f"executed: {params}")
+        fixture.skips = skips
+        tobiko.setup_fixture(fixture)
+
+    def setup_fixture(self):
+        # run validations
+        params = {name: True
+                  for name in self.skips}
+        LOG.info(f"Start executing Overcloud health checks: {params}.")
+        overcloud_health_checks(**params)
+        LOG.info(f"Overcloud health checks successfully executed: {params}.")
+
+    def cleanup_fixture(self):
+        self.skips = frozenset()
+
+
 @undercloud.skip_if_missing_undercloud
 class DisruptTripleoNodesTest(testtools.TestCase):
-
     """ HA Tests: run health check -> disruptive action -> health check
     disruptive_action: a function that runs some
-    disruptive scenarion on a overcloud"""
+    disruptive scenario on a overcloud"""
+
     def test_0vercloud_health_check(self):
-        overcloud_health_checks(skip_mac_table_size_test=False)
+        OvercloudHealthCheck.run_before(skip_mac_table_size_test=False)
 
     def test_hard_reboot_controllers_recovery(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.reset_all_controller_nodes()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
 
     def test_soft_reboot_computes_recovery(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.reset_all_compute_nodes(hard_reset=False)
         # verify VM status is updated after reboot
         nova.wait_for_all_instances_status('SHUTOFF')
         # start all VM instance
         # otherwise sidecar containers will not run after computes reboot
         nova.start_all_instances()
-        overcloud_health_checks(passive_checks_only=True)
+        OvercloudHealthCheck.run_after(passive_checks_only=True)
 
     # TODO(eolivare): the following test is skipped due to rhbz#1890895
     # def test_hard_reboot_computes_recovery(self):
-    #     overcloud_health_checks()
+    #     OvercloudHealthCheck.run_before()
     #     cloud_disruptions.reset_all_compute_nodes(hard_reset=True)
     #     # verify VM status is updated after reboot
     #     nova.wait_for_all_instances_status('SHUTOFF')
     #     # start all VM instance
     #     # otherwise sidecar containers will not run after computes reboot
     #     nova.start_all_instances()
-    #     overcloud_health_checks(passive_checks_only=True)
+    #     OvercloudHealthCheck.run_after(passive_checks_only=True)
 
     def test_reboot_controller_main_vip(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.reset_controller_main_vip()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
 
     def test_reboot_controller_non_main_vip(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.reset_controllers_non_main_vip()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
 
     def test_crash_controller_main_vip(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.crash_controller_main_vip()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
 
     def test_crash_controller_non_main_vip(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.crash_controllers_non_main_vip()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
 
     @pacemaker.skip_if_fencing_not_deployed
     def test_network_disruptor_main_vip(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.network_disrupt_controller_main_vip()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
         cloud_disruptions.network_undisrupt_controller_main_vip()
 
     # @pacemaker.skip_if_fencing_not_deployed
     # def test_network_disruptor_non_main_vip(self):
-    #     overcloud_health_checks()
+    #     OvercloudHealthCheck.run_before()
     #     cloud_disruptions.network_disrupt_controllers_non_main_vip()
-    #     overcloud_health_checks()
+    #     OvercloudHealthCheck.run_after()
     #     cloud_disruptions.network_undisrupt_controllers_non_main_vip()
 
     @neutron.skip_unless_is_ovn()
     def test_reset_ovndb_pcs_master_resource(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.reset_ovndb_pcs_master_resource()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
 
     @neutron.skip_unless_is_ovn()
     def test_reset_ovndb_pcs_resource(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.reset_ovndb_pcs_resource()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
 
     @neutron.skip_unless_is_ovn()
     def test_reset_ovndb_master_container(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.reset_ovndb_master_container()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
 
     def test_kill_rabbitmq_service_one_controller(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.kill_rabbitmq_service()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
 
     def test_kill_all_galera_services(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.kill_all_galera_services()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
 
     def test_remove_all_grastate_galera(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.remove_all_grastate_galera()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
 
     def test_remove_one_grastate_galera(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.remove_one_grastate_galera()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
 
     def test_request_galera_sst(self):
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_before()
         cloud_disruptions.request_galera_sst()
-        overcloud_health_checks()
+        OvercloudHealthCheck.run_after()
+
 
 # [..]
 # more tests to follow