x
/
tobiko
Code Issues Proposed changes

Test quorum number of controller nodes power on/off 

Test cloud recovery after two controller nodes has been
power off and then power on

The test is using undercloud Ironic REST APIs to
power off/on overcloud controller nodes. It is using
the 'not soft' way, therefore it should be considered
an hard shutdown operation.

This patch should verify BugZilla n. 1942094 [1]

[1] https://bugzilla.redhat.com/show_bug.cgi?id=1942094

Co-Authored-By: Slawek Kaplonski <skaplons@redhat.com>

Change-Id: Iae4c004e9744bac5da2fa444ab9ea9c406664b9b
This commit is contained in:
Federico Ressi 2021-04-12 10:45:43 +02:00
parent bd7533cda0
commit 680c026501
1 changed files with 33 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
tobiko/tests/faults/ha/test_cloud_recovery.py
View File

@ -1,5 +1,7 @@
from __future__ import absolute_import
import math
import random
import typing
from oslo_log import log
@ -8,6 +10,7 @@ import testtools
import tobiko
from tobiko.openstack import neutron
from tobiko.openstack import tests
from tobiko.openstack import topology
from tobiko.tests.faults.ha import cloud_disruptions
from tobiko.tripleo import pacemaker
from tobiko.tripleo import processes
@ -212,6 +215,36 @@ class DisruptTripleoNodesTest(testtools.TestCase):
cloud_disruptions.request_galera_sst()
OvercloudHealthCheck.run_after()
def test_controllers_shutdown(self):
all_nodes = topology.list_openstack_nodes(group='controller')
if len(all_nodes) < 3:
self.skipTest('It requires at least three controller nodes')
LOG.info("Ensure all controller nodes are running...")
for node in all_nodes:
node.power_on_overcloud_node()
LOG.info("Verify can create VMs before controllers power off...")
tests.test_server_creation()
quorum_level = math.ceil(0.5 * len(all_nodes))
assert quorum_level >= len(all_nodes) - quorum_level
nodes = random.sample(all_nodes, quorum_level)
LOG.info(f"Power off {quorum_level} random controller nodes: "
f"{[node.name for node in nodes]}")
for node in nodes:
node.power_off_overcloud_node()
random.shuffle(nodes)
LOG.info("Power on controller nodes: "
f"{[node.name for node in nodes]}")
for node in nodes:
node.power_on_overcloud_node()
LOG.info("Wait until pacemaker resources will be healthy again...")
check_pacemaker_resources_health()
LOG.info("Verify can create VMs after controllers power on...")
tests.test_server_creation()
# [..]
# more tests to follow