Merge "Improve faults tests' neutron agent health checks"

tobiko/openstack/tests/__init__.py

@@ -22,6 +22,8 @@ from tobiko.openstack.tests import _nova
 InvalidDBConnString = _neutron.InvalidDBConnString
 RAFTStatusError = _neutron.RAFTStatusError
 test_neutron_agents_are_alive = _neutron.test_neutron_agents_are_alive
+test_alive_agents_are_consistent_along_time = (
+    _neutron.test_alive_agents_are_consistent_along_time)
 test_ovn_dbs_validations = _neutron.test_ovn_dbs_validations
 test_ovs_bridges_mac_table_size = _neutron.test_ovs_bridges_mac_table_size
 test_ovs_namespaces_are_absent = _neutron.test_ovs_namespaces_are_absent

tobiko/openstack/tests/_neutron.py

@@ -94,6 +94,41 @@ def test_neutron_agents_are_alive(timeout=420., interval=5.) \
     return agents
 
 
+def test_alive_agents_are_consistent_along_time(previous_alive_agents=None):
+    test_case = tobiko.get_test_case()
+    if previous_alive_agents is None:
+        # the following dict of agents is obtained when:
+        # - the list_agents request is replied with 200
+        # - the list is not empty
+        # - no agents are dead
+        alive_agents = {agent['id']: agent
+                        for agent in test_neutron_agents_are_alive()}
+    else:
+        alive_agents = previous_alive_agents
+
+    for attempt in tobiko.retry(sleep_time=5., count=5):
+        agents = neutron.list_agents()
+        actual = {agent['id']: agent
+                  for agent in agents}
+
+        # any dead agents? If yes, fail now
+        dead_agents = agents.with_items(alive=False)
+        test_case.assertEqual(
+            [], dead_agents, "Some neutron agents died")
+
+        if len(actual) > len(alive_agents):
+            LOG.debug('Some new agents appeared! It seems not all the agents '
+                      'had been started yet, so let\'s restart this check')
+            return test_alive_agents_are_consistent_along_time(actual)
+
+        # any agent disappeared? If yes, fail now
+        test_case.assertEqual(
+            set(alive_agents), set(actual), 'Some agents disappeared')
+
+        if attempt.is_last:
+            break
+
+
 def ovn_dbs_vip_bindings(test_case):
     ovn_conn_str = get_ovn_db_connections()
     # ovn db sockets might be centrillized or distributed

tobiko/tests/faults/ha/test_cloud_recovery.py

@@ -45,7 +45,7 @@ def overcloud_health_checks(passive_checks_only=False,
     check_pacemaker_resources_health()
     check_overcloud_processes_health()
     nova.check_nova_services_health()
-    tests.test_neutron_agents_are_alive()
+    tests.test_alive_agents_are_consistent_along_time()
     if not passive_checks_only:
         # create a uniq stack
         check_vm_create()
@@ -265,7 +265,7 @@ class DisruptTripleoNodesTest(testtools.TestCase):
         cloud_disruptions.request_galera_sst()
         OvercloudHealthCheck.run_after()
 
-    @pytest.mark.flaky(reruns=3, reruns_delay=60)
+    @pytest.mark.flaky(reruns=0)
     def test_controllers_shutdown(self):
         OvercloudHealthCheck.run_before()
         cloud_disruptions.test_controllers_shutdown()

tobiko/tests/sanity/neutron/test_agents.py

@@ -17,8 +17,6 @@ from __future__ import absolute_import
 import pytest
 import testtools
 
-import tobiko
-from tobiko.openstack import neutron
 from tobiko.openstack import tests
 
 
@@ -29,16 +27,4 @@ class NeutronAgentTest(testtools.TestCase):
         tests.test_neutron_agents_are_alive()
 
     def test_alive_agents_are_consistent_along_time(self):
-        alive_agents = {agent['id']: agent
-                        for agent in tests.test_neutron_agents_are_alive()}
-        for attempt in tobiko.retry(sleep_time=5., count=5):
-            agents = neutron.list_agents()
-            actual = {agent['id']: agent
-                      for agent in agents}
-            self.assertEqual(set(alive_agents), set(actual),
-                             'Agents appeared or disappeared')
-            dead_agents = agents.with_items(alive=False)
-            self.assertEqual([], dead_agents,
-                             "Neutron agent(s) no more alive")
-            if attempt.is_last:
-                break
+        tests.test_alive_agents_are_consistent_along_time()