From 6589f06e4646ef0a3df3f6f65f92b3733d58e7fb Mon Sep 17 00:00:00 2001
From: Rodolfo Alonso Hernandez <ralonsoh@redhat.com>
Date: Wed, 11 Sep 2019 10:13:19 +0000
Subject: [PATCH] Re execute a test case if fixtures.TimeoutException is thrown

Since [1] was introduced, it's very frequent to have
"fixtures._fixtures.timeout.TimeoutException" exceptions during the
execution of UTs and FTs. Because the privsep includes the synchronized
decorator, the synchronization wait is done inside the privsep context.
This is prone to timeouts if the wait is too long.

Until we can reorder the decorators of ip_lib [2] or we can remove the
sync decorators [3], this patch can mitigate the errors in the CI.

Backporting for stable/stein as mentioned change [1] is in this branch

[1]https://review.opendev.org/#/c/631275/
[2]https://review.opendev.org/#/c/666853/
[3]https://review.opendev.org/#/c/657608/

Closes-Bug: #1843478

Change-Id: If865c4683645f9bd11f5e1b528bade0547505bfd
(cherry picked from commit 63715ea361ea94fb48842c83deef1e5850bcc44d)
---
 neutron/tests/base.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/neutron/tests/base.py b/neutron/tests/base.py
index 09ec56a0ec0..74a67828ff5 100644
--- a/neutron/tests/base.py
+++ b/neutron/tests/base.py
@@ -65,6 +65,8 @@ ETCDIR = os.path.join(ROOTDIR, 'etc')
 
 SUDO_CMD = 'sudo -n'
 
+TESTCASE_RETRIES = 3
+
 
 def etcdir(*p):
     return os.path.join(ETCDIR, *p)
@@ -172,10 +174,24 @@ class AttributeDict(dict):
 def _catch_timeout(f):
     @functools.wraps(f)
     def func(self, *args, **kwargs):
-        try:
-            return f(self, *args, **kwargs)
-        except eventlet.Timeout as e:
-            self.fail('Execution of this test timed out: %s' % e)
+        for idx in range(1, TESTCASE_RETRIES + 1):
+            try:
+                return f(self, *args, **kwargs)
+            except eventlet.Timeout as e:
+                self.fail('Execution of this test timed out: %s' % e)
+            # NOTE(ralonsoh): exception catch added due to the constant
+            # occurrences of this exception during FT and UT execution.
+            # This is due to [1]. Once the sync decorators are removed or the
+            # privsep ones are decorated by those ones (swap decorator
+            # declarations) this catch can be remove.
+            # [1] https://review.opendev.org/#/c/631275/
+            except fixtures.TimeoutException:
+                with excutils.save_and_reraise_exception() as ctxt:
+                    if idx < TESTCASE_RETRIES:
+                        msg = ('"fixtures.TimeoutException" during test case '
+                               'execution no %s; test case re-executed' % idx)
+                        self.addDetail('DietTestCase', msg)
+                        ctxt.reraise = False
     return func