Add OVN DBs synchronization validations

Verify all the OVN DBs running on the controller nodes are synchronized

Change-Id: I3c42f52a23de0eaf481117679c105a954329389b

tobiko/openstack/tests/__init__.py

@@ -20,6 +20,7 @@ from tobiko.openstack.tests import _neutron
 from tobiko.openstack.tests import _nova
 
 test_neutron_agents_are_alive = _neutron.test_neutron_agents_are_alive
+test_ovn_dbs_are_synchronized = _neutron.test_ovn_dbs_are_synchronized
 
 test_evacuable_server_creation = _nova.test_evacuable_server_creation
 test_server_creation = _nova.test_server_creation

tobiko/openstack/tests/_neutron.py

@@ -1,12 +1,16 @@
 from __future__ import absolute_import
 
 import json
+import re
 
 from oslo_log import log
 
 import tobiko
 from tobiko.openstack import neutron
+from tobiko.openstack import topology
 from tobiko.shell import sh
+from tobiko.tripleo import containers
+from tobiko.tripleo import pacemaker
 
 LOG = log.getLogger(__name__)
 
@@ -24,10 +28,26 @@ def get_osp_version():
 
 
 def is_ovn_configured():
-    from tobiko.tripleo import containers
     return containers.ovn_used_on_overcloud()
 
 
+def build_ovn_db_show_dict(ovn_db_show_str):
+    # returns a dictionary with OVN NB or OVN SB DB information
+    # each dict key is a section from the OVN DB command output
+    # each dict value is a list of strings
+    ovn_master_db_dict = {}
+    for line in ovn_db_show_str.splitlines():
+        if not re.match(r'^\s+', line):
+            # if line does not start with a space char, a new section will be
+            # started, i.e. a new dict entry
+            current_ovn_section = line.strip()
+            ovn_master_db_dict[current_ovn_section] = []
+        else:
+            ovn_master_db_dict[current_ovn_section].append(line.strip())
+
+    return ovn_master_db_dict
+
+
 def test_neutron_agents_are_alive(timeout=300., interval=5.):
     test_case = tobiko.get_test_case()
     for attempt in tobiko.retry(timeout=timeout, interval=interval):
@@ -73,3 +93,78 @@ def test_neutron_agents_are_alive(timeout=300., interval=5.):
 
         LOG.debug(f"All {len(agents)} Neutron agents are alive.")
         return agents
+
+
+def test_ovn_dbs_are_synchronized():
+    if not is_ovn_configured():
+        LOG.debug('OVN not configured. OVN DB sync validations skipped')
+        return
+
+    test_case = tobiko.get_test_case()
+
+    # declare commands
+    search_container_cmd = (
+        "sudo %s ps --format '{{.Names}}' -f name=ovn-dbs-bundle" %
+        containers.container_runtime_name)
+    container_cmd_prefix = ('sudo %s exec -uroot {container}' %
+                            containers.container_runtime_name)
+    ovndb_sync_cmd = ('ovs-appctl -t /var/run/openvswitch/{ovndb_ctl_file} '
+                      'ovsdb-server/sync-status')
+    ovndb_show_cmd = '{ovndb} show'
+    ovndb_ctl_file_dict = {'nb': 'ovnnb_db.ctl', 'sb': 'ovnsb_db.ctl'}
+    ovndb_dict = {'nb': 'ovn-nbctl', 'sb': 'ovn-sbctl'}
+    expected_state_active_str = 'state: active'
+    expected_state_backup_str = 'state: backup'
+
+    # use ovn master db as a reference
+    ovn_master_node_name = pacemaker.get_ovn_db_master_node()
+    test_case.assertEqual(1, len(ovn_master_node_name))
+    ovn_master_node = topology.get_openstack_node(ovn_master_node_name[0])
+    ovn_master_dbs_show_dict = {}
+    # obtained the container name
+    container_name = sh.execute(
+        search_container_cmd,
+        ssh_client=ovn_master_node.ssh_client).stdout.splitlines()[0]
+    for db in ('nb', 'sb'):
+        # check its synchronization is active
+        sync_cmd = (' '.join((container_cmd_prefix, ovndb_sync_cmd)).
+                    format(container=container_name,
+                           ovndb_ctl_file=ovndb_ctl_file_dict[db]))
+        sync_status = sh.execute(sync_cmd,
+                                 ssh_client=ovn_master_node.ssh_client).stdout
+        test_case.assertIn(expected_state_active_str, sync_status)
+        # obtain nb and sb show output
+        show_cmd = (' '.join((container_cmd_prefix, ovndb_show_cmd)).
+                    format(container=container_name, ovndb=ovndb_dict[db]))
+        ovn_db_show = sh.execute(
+            show_cmd, ssh_client=ovn_master_node.ssh_client).stdout
+        ovn_master_dbs_show_dict[db] = build_ovn_db_show_dict(ovn_db_show)
+
+    # ovn dbs are located on the controller nodes
+    for node in topology.list_openstack_nodes(group='controller'):
+        if node.name == ovn_master_node.name:
+            # master node is the reference and do not need to be checked again
+            continue
+        container_name = sh.execute(
+            search_container_cmd,
+            ssh_client=node.ssh_client).stdout.splitlines()[0]
+        # verify ovn nb and sb dbs are synchronized
+        ovn_dbs_show_dict = {}
+        for db in ('nb', 'sb'):
+            # check its synchronization is active
+            sync_cmd = (' '.join((container_cmd_prefix, ovndb_sync_cmd)).
+                        format(container=container_name,
+                               ovndb_ctl_file=ovndb_ctl_file_dict[db]))
+            sync_status = sh.execute(sync_cmd,
+                                     ssh_client=node.ssh_client).stdout
+            test_case.assertIn(expected_state_backup_str, sync_status)
+            # obtain nb and sb show output
+            show_cmd = (' '.join((container_cmd_prefix, ovndb_show_cmd)).
+                        format(container=container_name, ovndb=ovndb_dict[db]))
+            ovn_db_show = sh.execute(
+                show_cmd, ssh_client=node.ssh_client).stdout
+            ovn_dbs_show_dict[db] = build_ovn_db_show_dict(ovn_db_show)
+            test_case.assertEqual(ovn_dbs_show_dict[db],
+                                  ovn_master_dbs_show_dict[db])
+
+    LOG.info("All OVN DBs are synchronized")

tobiko/tests/faults/ha/test_cloud_recovery.py

@@ -27,6 +27,7 @@ def overcloud_health_checks(passive_checks_only=False):
     containers.assert_all_tripleo_containers_running()
     containers.assert_equal_containers_state()
     containers.run_container_config_validations()
+    tests.test_ovn_dbs_are_synchronized()
     validations.run_post_deployment_validations()