Merge "Extend podified HA health checks"

tobiko/openstack/tests/__init__.py

@@ -23,6 +23,7 @@ test_neutron_agents_are_alive = _neutron.test_neutron_agents_are_alive
 test_alive_agents_are_consistent_along_time = (
     _neutron.test_alive_agents_are_consistent_along_time)
 test_ovn_dbs_validations = _neutron.test_ovn_dbs_validations
+test_ovn_dbs_are_synchronized = _neutron.test_ovn_dbs_are_synchronized
 test_ovs_bridges_mac_table_size = _neutron.test_ovs_bridges_mac_table_size
 test_ovs_namespaces_are_absent = _neutron.test_ovs_namespaces_are_absent
 test_ovs_interfaces_are_absent = _neutron.test_ovs_interfaces_are_absent

tobiko/openstack/tests/_neutron.py

@@ -183,6 +183,21 @@ def ovn_dbs_are_synchronized(test_case):
         LOG.debug('OVN %s database dump: %s', ovndb, dumps[ovndb])
 
 
+def test_ovn_dbs_are_synchronized():
+    """Validate OVN DBs are synchronized across all OVN DB hosts.
+
+    Unlike test_ovn_dbs_validations(), this check does not inspect the OVN DB
+    VIP listening sockets on controller nodes, so it is safe to run on
+    podified deployments where OVN runs inside ovsdbserver pods.
+    """
+    if not neutron.has_ovn():
+        LOG.debug('OVN not configured. OVN DB sync validations skipped')
+        return
+
+    test_case = tobiko.get_test_case()
+    ovn_dbs_are_synchronized(test_case)
+
+
 def test_ovn_dbs_validations():
     if not neutron.has_ovn():
         LOG.debug('OVN not configured. OVN DB sync validations skipped')

tobiko/podified/__init__.py

@@ -27,6 +27,7 @@ EDPM_COMPUTE_GROUP = _openshift.EDPM_COMPUTE_GROUP
 EDPM_NETWORKER_GROUP = _openshift.EDPM_NETWORKER_GROUP
 EDPM_OTHER_GROUP = _openshift.EDPM_OTHER_GROUP
 OSP_CONTROLPLANE = _openshift.OSP_CONTROLPLANE
+OSP_DP_DEPLOYMENT = _openshift.OSP_DP_DEPLOYMENT
 ALL_COMPUTES_GROUP_NAME = _topology.ALL_COMPUTES_GROUP_NAME
 
 PodifiedTopology = _topology.PodifiedTopology
@@ -52,6 +53,7 @@ wait_for_transporturl_setup_complete = (
     _openshift.wait_for_transporturl_setup_complete)
 get_rabbitmq_user_labels = _openshift.get_rabbitmq_user_labels
 list_nodeset_names = _openshift.list_nodeset_names
+list_dataplane_deployment_names = _openshift.list_dataplane_deployment_names
 trigger_edpm_deployment = _openshift.trigger_edpm_deployment
 wait_for_edpm_deployment_ready = _openshift.wait_for_edpm_deployment_ready
 get_ovndbcluter = _openshift.get_ovndbcluter

tobiko/podified/_openshift.py

@@ -368,6 +368,12 @@ def list_nodeset_names():
                 oc.selector(OSP_DP_NODESET).objects()]
 
 
+def list_dataplane_deployment_names():
+    with project_context():
+        return [obj.name() for obj in
+                oc.selector(OSP_DP_DEPLOYMENT).objects()]
+
+
 def trigger_edpm_deployment(nodeset_names):
     deploy_name = f"rotation-deploy-{random.randrange(1000000)}"
     deploy_def = {

tobiko/tests/faults/ha/test_cloud_recovery.py

@@ -127,13 +127,17 @@ class OvercloudHealthCheck(tobiko.SharedFixture):
         cls.run(after=True, **params)
 
     @classmethod
-    def run(cls, after: bool, **params):
-        fixture = tobiko.get_fixture(cls)
-        params.setdefault('passive_checks_only', False)
+    def set_version_dependent_skips(cls, params):
         # In version OSP17.0 and highier,
         # 'test_ovs_bridges_mac_table_size()' test can run.
         if topology.verify_osp_version('17.0', lower=True):
             params.setdefault('skip_mac_table_size_test', True)
+
+    @classmethod
+    def run(cls, after: bool, **params):
+        fixture = tobiko.get_fixture(cls)
+        params.setdefault('passive_checks_only', False)
+        cls.set_version_dependent_skips(params)
         skips = frozenset(k for k, v in params.items() if v)
         if after or skips < fixture.skips:
             # Force re-check

tobiko/tests/faults/podified/ha/test_cloud_recovery.py

@@ -19,40 +19,114 @@ import random
 from oslo_log import log
 import testtools
 
+import tobiko
+from tobiko import config
 from tobiko.tests.faults.ha import test_cloud_recovery
 from tobiko.tests.faults.podified.ha import cloud_disruptions
 from tobiko.tests.faults.podified.ha import rotate_galera_root_password
 from tobiko.openstack import tests
+from tobiko.openstack import topology
 from tobiko import podified
 from tobiko.openstack import nova
+from tobiko.podified import containers as podified_containers
 from tobiko.shell import sh
 
 
+CONF = config.CONF
 LOG = log.getLogger(__name__)
 
 
-def podified_health_checks():
+def check_edpm_deployment_health():
+    """Wait for all EDPM data plane deployments to become ready."""
+    names = podified.list_dataplane_deployment_names()
+    if not names:
+        tobiko.fail(f'No {podified.OSP_DP_DEPLOYMENT} found in project '
+                    f'{CONF.tobiko.podified.osp_project}')
+    LOG.info('Waiting for EDPM data plane deployments to be ready: %r', names)
+    for name in names:
+        podified.wait_for_edpm_deployment_ready(name)
+
+
+def check_podified_containers_health():
+    """Check EDPM host containers (nova_compute and OVN) are running."""
+    podified_containers.list_node_containers.cache_clear()
+    podified_containers.assert_containers_running(
+        podified.EDPM_COMPUTE_GROUP, ['nova_compute'])
+    podified_containers.assert_ovn_containers_running()
+    podified_containers.assert_equal_containers_state()
+
+
+def _is_ovn_bgp_agent_running():
+    groups = topology.get_openstack_topology().groups
+    for group in (podified.EDPM_NETWORKER_GROUP, podified.EDPM_COMPUTE_GROUP):
+        if group not in groups:
+            continue
+        if podified_containers.assert_containers_running(
+                group, ['ovn_bgp_agent'], bool_check=True):
+            return True
+    return False
+
+
+def check_vm_create():
+    tests.test_server_creation()
+    if _is_ovn_bgp_agent_running():
+        try:
+            node = topology.find_openstack_node(
+                group=podified.EDPM_NETWORKER_GROUP)
+        except topology.NoSuchOpenStackTopologyNodeGroup:
+            node = topology.find_openstack_node(
+                group=podified.EDPM_COMPUTE_GROUP)
+        expose_tenant_networks = topology.get_config_setting(
+            'bgp-agent.conf', node.ssh_client, 'expose_tenant_networks')
+        if (expose_tenant_networks and
+                expose_tenant_networks.lower() == 'true'):
+            tests.test_server_creation_no_fip()
+
+
+def podified_health_checks(passive_checks_only=False, **_kwargs):
+    # Extra skip flags (e.g. skip_mac_table_size_test) are accepted for
+    # compatibility with OvercloudHealthCheck but ignored here: the OVS
+    # mac-table-size validation relies on TripleO config paths that do not
+    # exist on EDPM nodes.
     podified.assert_ocp_pods_running()
+    check_edpm_deployment_health()
     nova.check_nova_services_health()
     tests.test_alive_agents_are_consistent_along_time()
-    # create a unique stack that will be cleaned up at the end of each test
-    # TODO(eolivare) add tests.test_server_creation_no_fip() when BGP is
-    # configured with expose_tenant_networks
-    tests.test_server_creation()
-    nova.action_on_all_instances('active')
-    nova.check_virsh_domains_running()
+    if not passive_checks_only:
+        # create a unique stack that will be cleaned up at the end of each test
+        check_vm_create()
+        nova.action_on_all_instances('active')
+        nova.check_virsh_domains_running()
+    check_podified_containers_health()
+    tests.test_ovn_dbs_are_synchronized()
     test_cloud_recovery.octavia_health_checks()
 
 
 class PodifiedCloudHealthCheck(test_cloud_recovery.OvercloudHealthCheck):
+
+    @classmethod
+    def set_version_dependent_skips(cls, params):
+        # The parent implementation calls verify_osp_version('17.0'), which
+        # reads /etc/rhosp-release from a node of the 'controller' group. On
+        # podified that group holds OCP control plane nodes, where neither
+        # that file nor the nova_conductor container nor nova-manage exist.
+        # The three failed lookups are swallowed, but they are logged as
+        # command failures and show up as spurious errors in the job logs.
+        # The mac-table-size validation they gate is not applicable to EDPM
+        # nodes anyway, so do not probe the version at all here.
+        pass
+
     def setup_fixture(self):
-        LOG.info("Start executing Podified health checks.")
+        # run validations
+        params = {name: True for name in self.skips}
+        LOG.info(f"Start executing Podified health checks: {params}.")
         try:
-            podified_health_checks()
+            podified_health_checks(**params)
         except Exception:
             LOG.exception("Podified health checks failed.")
             raise
-        LOG.info("Podified health checks successfully executed.")
+        LOG.info(
+            f"Podified health checks successfully executed: {params}.")
 
 
 @podified.skip_if_not_podified

tobiko/tests/unit/openstack/tests/test_neutron.py

@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from unittest import mock
 
 from tobiko.openstack import tests
+from tobiko.openstack.tests import _neutron
 from tobiko.tests.unit import openstack
 
 
@@ -53,3 +54,18 @@ class NeutronAgentTest(openstack.OpenstackTest):
                          '        "alive": false\n'
                          "    }\n"
                          "]\n", str(ex))
+
+
+class OvnDbsSyncTest(openstack.OpenstackTest):
+
+    def test_skips_when_ovn_not_configured(self):
+        self.patch(_neutron.neutron, 'has_ovn', return_value=False)
+        sync = self.patch(_neutron, 'ovn_dbs_are_synchronized')
+        tests.test_ovn_dbs_are_synchronized()
+        sync.assert_not_called()
+
+    def test_runs_sync_check_when_ovn_configured(self):
+        self.patch(_neutron.neutron, 'has_ovn', return_value=True)
+        sync = self.patch(_neutron, 'ovn_dbs_are_synchronized')
+        tests.test_ovn_dbs_are_synchronized()
+        sync.assert_called_once()