From 63715ea361ea94fb48842c83deef1e5850bcc44d Mon Sep 17 00:00:00 2001
From: Rodolfo Alonso Hernandez <ralonsoh@redhat.com>
Date: Wed, 11 Sep 2019 10:13:19 +0000
Subject: [PATCH] Re execute a test case if fixtures.TimeoutException is thrown

Since [1] was introduced, it's very frequent to have
"fixtures._fixtures.timeout.TimeoutException" exceptions during the
execution of UTs and FTs. Because the privsep includes the synchronized
decorator, the synchronization wait is done inside the privsep context.
This is prone to timeouts if the wait is too long.

Until we can reorder the decorators of ip_lib [2] or we can remove the
sync decorators [3], this patch can mitigate the errors in the CI.

[1]https://review.opendev.org/#/c/631275/
[2]https://review.opendev.org/#/c/666853/
[3]https://review.opendev.org/#/c/657608/

Closes-Bug: #1843478

Change-Id: If865c4683645f9bd11f5e1b528bade0547505bfd
---
 neutron/tests/base.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/neutron/tests/base.py b/neutron/tests/base.py
index e9c56f7d82a..36a4a9b7531 100644
--- a/neutron/tests/base.py
+++ b/neutron/tests/base.py
@@ -66,6 +66,8 @@ ETCDIR = os.path.join(ROOTDIR, 'etc')
 
 SUDO_CMD = 'sudo -n'
 
+TESTCASE_RETRIES = 3
+
 
 def etcdir(*p):
     return os.path.join(ETCDIR, *p)
@@ -173,10 +175,24 @@ class AttributeDict(dict):
 def _catch_timeout(f):
     @functools.wraps(f)
     def func(self, *args, **kwargs):
-        try:
-            return f(self, *args, **kwargs)
-        except eventlet.Timeout as e:
-            self.fail('Execution of this test timed out: %s' % e)
+        for idx in range(1, TESTCASE_RETRIES + 1):
+            try:
+                return f(self, *args, **kwargs)
+            except eventlet.Timeout as e:
+                self.fail('Execution of this test timed out: %s' % e)
+            # NOTE(ralonsoh): exception catch added due to the constant
+            # occurrences of this exception during FT and UT execution.
+            # This is due to [1]. Once the sync decorators are removed or the
+            # privsep ones are decorated by those ones (swap decorator
+            # declarations) this catch can be remove.
+            # [1] https://review.opendev.org/#/c/631275/
+            except fixtures.TimeoutException:
+                with excutils.save_and_reraise_exception() as ctxt:
+                    if idx < TESTCASE_RETRIES:
+                        msg = ('"fixtures.TimeoutException" during test case '
+                               'execution no %s; test case re-executed' % idx)
+                        self.addDetail('DietTestCase', msg)
+                        ctxt.reraise = False
     return func