x/tobiko
Code Issues Proposed changes

restart ovn faults tests

Browse Source 
Change-Id: I1b47198148609971e0f4aa0451b26b6ed50925cf
This commit is contained in:
pinikomarov 2020-06-14 04:23:09 +03:00
parent f530416164
commit 52491378e6
4 changed files with 77 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
tobiko/tests/faults/ha/cloud_disruptions.py
View File

@ -9,6 +9,7 @@ from tobiko.openstack import topology
from tobiko.tripleo import topology as tripleo_topology
from tobiko.openstack import keystone
from tobiko.tripleo import pacemaker
from tobiko.tripleo import containers
from oslo_log import log
from tobiko.tests.faults.ha import test_cloud_recovery
@ -33,6 +34,7 @@ network_disruption = """
undisrupt_network = """
sudo iptables-restore /root/working.iptables.rules
"""
ovn_db_pcs_resource_restart = """sudo pcs resource restart ovn-dbs-bundle"""
def get_node(node_name):
@ -203,3 +205,22 @@ def reset_all_compute_nodes(hard_reset=False):
compute_checked = sh.execute("hostname", ssh_client=compute.ssh_client,
expect_exit_status=None).stdout
LOG.info('{} is up '.format(compute_checked))
def reset_ovndb_master_resource():
"""restart ovndb pacemaker resource"""
disrupt_node('controller-0', disrupt_method=ovn_db_pcs_resource_restart)
def reset_ovndb_master_container():
"""get and restart the ovndb master container
use of partial name : resource: ovn-dbs-bundle-0 =>
container: ovn-dbs-bundle-podman-2"""
node = pacemaker.get_overcloud_nodes_running_pcs_resource(
resource_type='(ocf::ovn:ovndb-servers):', resource_state='Master')[0]
resource = pacemaker.get_overcloud_resource(
resource_type='(ocf::ovn:ovndb-servers):', resource_state='Master')
resource = resource[0].rsplit('-', 1)[0]
containers.action_on_container('restart',
partial_container_name=resource,
container_host=node)

10
tobiko/tests/faults/ha/test_cloud_recovery.py
View File

@ -122,6 +122,16 @@ class RebootTripleoNodesTest(testtools.TestCase):
cloud_disruptions.network_disrupt_controllers_non_main_vip()
overcloud_health_checks()
cloud_disruptions.network_undisrupt_controllers_non_main_vip()
def test_reset_ovndb_master_resource(self):
overcloud_health_checks()
cloud_disruptions.reset_ovndb_master_resource()
overcloud_health_checks()
def test_reset_ovndb_master_container(self):
overcloud_health_checks()
cloud_disruptions.reset_ovndb_master_container()
overcloud_health_checks()
# [..]
# more tests to follow
# run health checks

17
tobiko/tripleo/containers.py
View File

@ -262,11 +262,18 @@ def list_containers_objects_df():
return containers_objects_list_df
def get_overcloud_container(container_name=None, container_host=None):
def get_overcloud_container(container_name=None, container_host=None,
partial_container_name=None):
"""gets an container object by name on specified host
container"""
con_obj_df = list_containers_objects_df()
if container_host:
if partial_container_name and container_host:
con_obj_df = con_obj_df[con_obj_df['container_name'].str.contains(
partial_container_name)]
contaniner_obj = con_obj_df.query(
'container_host == "{container_host}"'.format(
container_host=container_host))['container_object']
elif container_host:
contaniner_obj = con_obj_df.query(
'container_name == "{container_name}"'
' and container_host == "{container_host}"'.
@ -283,12 +290,14 @@ def get_overcloud_container(container_name=None, container_host=None):
def action_on_container(action,
container_name=None, container_host=None):
container_name=None, container_host=None,
partial_container_name=None):
"""take a container snd preform an action on it
actions are as defined in : podman/libs/containers.py:14/164"""
container = get_overcloud_container(
container_name=container_name,
container_host=container_host)
container_host=container_host,
partial_container_name=partial_container_name)
# we get the specified action as function from podman lib
if container_runtime_module == podman:
container_function = getattr(

34
tobiko/tripleo/pacemaker.py
View File

@ -234,7 +234,7 @@ def get_overcloud_nodes_running_pcs_resource(resource=None,
resource_state=None):
"""
Check what nodes are running the specified resource/type/state
resource/type/state: exact str of a process name as seen in pcs status
resource/type/state: exact str of a resource name as seen in pcs status
:return: list of overcloud nodes
"""
pcs_df = get_pcs_resources_table()
@ -254,3 +254,35 @@ def get_overcloud_nodes_running_pcs_resource(resource=None,
pcs_df_query_resource_type = pcs_df.query(
'resource_type=="{}"'.format(resource_type))
return pcs_df_query_resource_type['overcloud_node'].unique().tolist()
def get_resource_master_node(resource_type=None):
get_overcloud_nodes_running_pcs_resource(
resource_type=resource_type, resource_state='Master')
def get_ovn_db_master_node():
get_overcloud_nodes_running_pcs_resource(
resource_type='(ocf::ovn:ovndb-servers):', resource_state='Master')
def get_overcloud_resource(resource_type=None,
resource_state=None):
"""
Check what nodes are running the specified resource/type/state
resource/type/state: exact str of a resource name as seen in pcs status
:return: list of overcloud nodes
"""
pcs_df = get_pcs_resources_table()
if resource_type and resource_state:
pcs_df_query_resource_type_state = pcs_df.query(
'resource_type=="{}" and resource_state=="{}"'.format(
resource_type, resource_state))
return pcs_df_query_resource_type_state[
'resource'].unique().tolist()
if resource_type and not resource_state:
pcs_df_query_resource_type = pcs_df.query(
'resource_type=="{}"'.format(resource_type))
return pcs_df_query_resource_type['resource'].unique().tolist()