From c9dbbd41921ba5e81c986e78791c5ebfa5816c93 Mon Sep 17 00:00:00 2001
From: Szymon Wroblewski <szymon.wroblewski@ovhcloud.com>
Date: Fri, 2 Sep 2022 11:26:40 +0200
Subject: [PATCH] Retry connections to Nova

Sometimes Neutron is failing to send notification to Nova
due to timeout, refused connection or another HTTP error.
Retry send in those cases.

Conflicts:
    neutron/notifiers/nova.py
    neutron/tests/unit/notifiers/test_nova.py

Closes-Bug: #1987780
Change-Id: Iaaccec770484234b704f70f3c144efac4d8ffba0
(cherry picked from commit cd475f9af898b81d98b3e0d3f55b94ea653c193c)
(cherry picked from commit b5e9148cc7311080ba1b2a410145949c1adaa0ca)
(cherry picked from commit 49f49bc2bf3b477df00a92e25a8de66221ec8ee6)
---
 neutron/notifiers/nova.py                 | 13 +++++++++++++
 neutron/tests/unit/notifiers/test_nova.py | 10 ++++++++++
 2 files changed, 23 insertions(+)

diff --git a/neutron/notifiers/nova.py b/neutron/notifiers/nova.py
index 3331f7d5f09..7a68e29b416 100644
--- a/neutron/notifiers/nova.py
+++ b/neutron/notifiers/nova.py
@@ -13,6 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+
+from keystoneauth1 import exceptions as ks_exceptions
 from keystoneauth1 import loading as ks_loading
 from neutron_lib.callbacks import events
 from neutron_lib.callbacks import registry
@@ -29,6 +31,7 @@ from oslo_context import context as common_context
 from oslo_log import log as logging
 from oslo_utils import uuidutils
 from sqlalchemy.orm import attributes as sql_attr
+import tenacity
 
 from neutron.notifiers import batch_notifier
 
@@ -242,6 +245,12 @@ class Notifier(object):
              'tag': port.id})
         self.send_port_status(None, None, port)
 
+    @tenacity.retry(
+        retry=tenacity.retry_if_exception_type(
+            ks_exceptions.RetriableConnectionFailure),
+        wait=tenacity.wait_exponential(multiplier=0.01, max=1),
+        stop=tenacity.stop_after_delay(1),
+        after=tenacity.after_log(LOG, logging.DEBUG))
     def send_events(self, batched_events):
         LOG.debug("Sending events: %s", batched_events)
         novaclient = self._get_nova_client()
@@ -251,6 +260,10 @@ class Notifier(object):
         except nova_exceptions.NotFound:
             LOG.debug("Nova returned NotFound for event: %s",
                       batched_events)
+        except ks_exceptions.RetriableConnectionFailure:
+            raise
+            # next clause handles all exceptions
+            # so reraise for retry decorator
         except Exception:
             LOG.exception("Failed to notify nova on events: %s",
                           batched_events)
diff --git a/neutron/tests/unit/notifiers/test_nova.py b/neutron/tests/unit/notifiers/test_nova.py
index 4b4607e3404..a1a76b8fd61 100644
--- a/neutron/tests/unit/notifiers/test_nova.py
+++ b/neutron/tests/unit/notifiers/test_nova.py
@@ -15,6 +15,8 @@
 
 
 import mock
+
+from keystoneauth1 import exceptions as ks_exc
 from neutron_lib import constants as n_const
 from neutron_lib import context as n_ctx
 from neutron_lib import exceptions as n_exc
@@ -246,6 +248,14 @@ class TestNovaNotify(base.BaseTestCase):
             nova_exceptions.NotFound)
         self.nova_notifier.send_events([])
 
+    @mock.patch('novaclient.client.Client')
+    def test_nova_send_events_raises_connect_exc(self, mock_client):
+        create = mock_client().server_external_events.create
+        create.side_effect = (
+            ks_exc.ConnectFailure, ks_exc.ConnectTimeout, [])
+        self.nova_notifier.send_events([])
+        self.assertEqual(3, create.call_count)
+
     @mock.patch('novaclient.client.Client')
     def test_nova_send_events_raises(self, mock_client):
         mock_client.server_external_events.create.return_value = Exception