From 680c026501266275088801f426725a7bc3cff758 Mon Sep 17 00:00:00 2001
From: Federico Ressi <fressi@redhat.com>
Date: Mon, 12 Apr 2021 10:45:43 +0200
Subject: [PATCH] Test quorum number of controller nodes power on/off

Test cloud recovery after two controller nodes has been
power off and then power on

The test is using undercloud Ironic REST APIs to
power off/on overcloud controller nodes. It is using
the 'not soft' way, therefore it should be considered
an hard shutdown operation.

This patch should verify BugZilla n. 1942094 [1]

[1] https://bugzilla.redhat.com/show_bug.cgi?id=1942094

Co-Authored-By: Slawek Kaplonski <skaplons@redhat.com>

Change-Id: Iae4c004e9744bac5da2fa444ab9ea9c406664b9b
---
 tobiko/tests/faults/ha/test_cloud_recovery.py | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/tobiko/tests/faults/ha/test_cloud_recovery.py b/tobiko/tests/faults/ha/test_cloud_recovery.py
index 7575c2bf9..3981ca344 100644
--- a/tobiko/tests/faults/ha/test_cloud_recovery.py
+++ b/tobiko/tests/faults/ha/test_cloud_recovery.py
@@ -1,5 +1,7 @@
 from __future__ import absolute_import
 
+import math
+import random
 import typing
 
 from oslo_log import log
@@ -8,6 +10,7 @@ import testtools
 import tobiko
 from tobiko.openstack import neutron
 from tobiko.openstack import tests
+from tobiko.openstack import topology
 from tobiko.tests.faults.ha import cloud_disruptions
 from tobiko.tripleo import pacemaker
 from tobiko.tripleo import processes
@@ -212,6 +215,36 @@ class DisruptTripleoNodesTest(testtools.TestCase):
         cloud_disruptions.request_galera_sst()
         OvercloudHealthCheck.run_after()
 
+    def test_controllers_shutdown(self):
+        all_nodes = topology.list_openstack_nodes(group='controller')
+        if len(all_nodes) < 3:
+            self.skipTest('It requires at least three controller nodes')
+        LOG.info("Ensure all controller nodes are running...")
+        for node in all_nodes:
+            node.power_on_overcloud_node()
+
+        LOG.info("Verify can create VMs before controllers power off...")
+        tests.test_server_creation()
+
+        quorum_level = math.ceil(0.5 * len(all_nodes))
+        assert quorum_level >= len(all_nodes) - quorum_level
+        nodes = random.sample(all_nodes, quorum_level)
+        LOG.info(f"Power off {quorum_level} random controller nodes: "
+                 f"{[node.name for node in nodes]}")
+        for node in nodes:
+            node.power_off_overcloud_node()
+
+        random.shuffle(nodes)
+        LOG.info("Power on controller nodes: "
+                 f"{[node.name for node in nodes]}")
+        for node in nodes:
+            node.power_on_overcloud_node()
+
+        LOG.info("Wait until pacemaker resources will be healthy again...")
+        check_pacemaker_resources_health()
+        LOG.info("Verify can create VMs after controllers power on...")
+        tests.test_server_creation()
+
 
 # [..]
 # more tests to follow