Retry on ServerFault in call_and_ignore_notfound_exc()

This makes us tolerate up to three ServerFault errors while trying to
do things like clean up resources during tearDown().

Change-Id: I3b2dac90fd6c71d66506d33aa5e35bb362d9bf87
Related-Bug: #1897907

tempest/lib/common/utils/test_utils.py

@@ -80,10 +80,19 @@ def find_test_caller():
 
 def call_and_ignore_notfound_exc(func, *args, **kwargs):
     """Call the given function and pass if a `NotFound` exception is raised."""
-    try:
-        return func(*args, **kwargs)
-    except exceptions.NotFound:
-        pass
+    attempt = 0
+    while True:
+        attempt += 1
+        try:
+            return func(*args, **kwargs)
+        except exceptions.NotFound:
+            return
+        except exceptions.ServerFault:
+            # NOTE(danms): Tolerate three ServerFault exceptions while trying
+            # to do this thing, and after that, assume it's legit.
+            if attempt >= 3:
+                raise
+            LOG.warning('Got ServerFault while running %s, retrying...', func)
 
 
 def call_until_true(func, duration, sleep_for, *args, **kwargs):

tempest/tests/lib/common/utils/test_test_utils.py

@@ -74,6 +74,17 @@ class TestTestUtils(base.TestCase):
         self.assertRaises(ValueError, test_utils.call_and_ignore_notfound_exc,
                           raise_value_error)
 
+    def test_call_and_ignore_notfound_exc_when_serverfault_raised(self):
+        calls = []
+
+        def raise_serverfault():
+            calls.append('call')
+            raise exceptions.ServerFault()
+        self.assertRaises(exceptions.ServerFault,
+                          test_utils.call_and_ignore_notfound_exc,
+                          raise_serverfault)
+        self.assertEqual(3, len(calls))
+
     def test_call_and_ignore_notfound_exc(self):
         m = mock.Mock(return_value=42)
         args, kwargs = (1,), {'1': None}