Merge "adding messaging node possibility"

tobiko/tests/faults/ha/cloud_disruptions.py

@@ -3,6 +3,7 @@ from __future__ import absolute_import
 
 import re
 import time
+import random
 
 from oslo_log import log
 
@@ -35,6 +36,7 @@ undisrupt_network = """
  sudo iptables-restore /root/working.iptables.rules
 """
 ovn_db_pcs_resource_restart = """sudo pcs resource restart ovn-dbs-bundle"""
+kill_rabbit = """sudo kill -9 $(pgrep beam.smp)"""
 
 
 def get_node(node_name):
@@ -304,6 +306,25 @@ def reset_ovndb_master_container():
                                    container_host=node)
 
 
+def kill_rabbitmq_service():
+    """kill a rabbit process on a random controller,
+    check in pacemaker it is down"""
+    if 'messaging' in topology.list_openstack_node_groups():
+        group = 'messaging'
+    else:
+        group = 'controller'
+    nodes = topology.list_openstack_nodes(group=group)
+    node = random.choice(nodes)
+    sh.execute(kill_rabbit, ssh_client=node.ssh_client)
+    LOG.info('kill rabbit: {} on server: {}'.format(kill_rabbit,
+             node.name))
+    retry = tobiko.retry(timeout=30, interval=5)
+    for _ in retry:
+        if not(pacemaker.PacemakerResourcesStatus().
+               rabbitmq_resource_healthy()):
+            return
+
+
 def evac_failover_compute(compute_host, failover_type=sh.hard_reset_method):
     """disrupt a compute, to trigger it's instance-HA evacuation
     failover_type=hard_reset_method etc.."""

tobiko/tests/faults/ha/test_cloud_recovery.py

@@ -119,6 +119,12 @@ class DisruptTripleoNodesTest(testtools.TestCase):
         cloud_disruptions.reset_ovndb_master_container()
         overcloud_health_checks()
 
+    def test_kill_rabbitmq_service_one_controller(self):
+        overcloud_health_checks()
+        cloud_disruptions.kill_rabbitmq_service()
+        overcloud_health_checks()
+
+
 # [..]
 # more tests to follow
 # run health checks