From 11daac03bf4d33c77c7bc6f037bc16972ad64ea4 Mon Sep 17 00:00:00 2001
From: melanie witt <melwittt@gmail.com>
Date: Wed, 11 Mar 2020 02:26:52 +0000
Subject: [PATCH] Add config option for neutron client retries

Nova can occasionally fail to carry out server actions which require
calls to neutron API if haproxy happens to close a connection after
idle time if an incoming request attempts to re-use the connection
while it is being torn down.

In order to be more resilient to this scenario, we can add a config
option for neutron client to retry requests, similar to our existing
CONF.cinder.http_retries and CONF.glance.num_retries options.

Because we create our neutron client [1] using a keystoneauth1 session
[2], we can set the 'connect_retries' keyword argument to let
keystoneauth1 handle connection retries.

Conflicts:
	nova/conf/neutron.py
NOTE(s10): conflict is due to Id7c2f0b53c8871ff47a836ec4c324c8cce430b79
not being in Queens.

Closes-Bug: #1866937

[1] https://github.com/openstack/nova/blob/57459c3429ce62975cebd0cd70936785bdf2f3a4/nova/network/neutron.py#L226-L237
[2] https://docs.openstack.org/keystoneauth/latest/api/keystoneauth1.session.html#keystoneauth1.session.Session

Change-Id: Ifb3afb13aff7e103c2e80ade817d0e63b624604a
(cherry picked from commit 0e34ed9733e3f23d162e3e428795807386abf1cb)
(cherry picked from commit 71971c206292232fff389dedbf412d780f0a557a)
(cherry picked from commit a96e7ab83066bee0a13a54070aab988396bae320)
(cherry picked from commit 6bed39cd6e77744ca4bbad9dba3f68c78cdc1ec0)
---
 nova/conf/neutron.py                               | 14 ++++++++++++++
 nova/network/neutronv2/api.py                      |  3 ++-
 nova/tests/unit/network/test_neutronv2.py          |  9 +++++++++
 ...eutron-connection-retries-c276010afe238abc.yaml | 12 ++++++++++++
 4 files changed, 37 insertions(+), 1 deletion(-)
 create mode 100644 releasenotes/notes/neutron-connection-retries-c276010afe238abc.yaml

diff --git a/nova/conf/neutron.py b/nova/conf/neutron.py
index 48cadc77cea9..d828f8ca80ed 100644
--- a/nova/conf/neutron.py
+++ b/nova/conf/neutron.py
@@ -76,6 +76,20 @@ Neutron for extensions.  After this number of seconds the next time Nova
 needs to create a resource in Neutron it will requery Neutron for the
 extensions that it has loaded.  Setting value to 0 will refresh the
 extensions with no wait.
+"""),
+    cfg.IntOpt('http_retries',
+               default=3,
+               min=0,
+               help="""
+Number of times neutronclient should retry on any failed http call.
+
+0 means connection is attempted only once. Setting it to any positive integer
+means that on failure connection is retried that many times e.g. setting it
+to 3 means total attempts to connect will be 4.
+
+Possible values:
+
+* Any integer value. 0 means connection is attempted only once
 """),
 ]
 
diff --git a/nova/network/neutronv2/api.py b/nova/network/neutronv2/api.py
index ba2911ae060b..6ec851f5976f 100644
--- a/nova/network/neutronv2/api.py
+++ b/nova/network/neutronv2/api.py
@@ -163,7 +163,8 @@ def get_client(context, admin=False):
 
     client_args = dict(session=_SESSION,
                        auth=auth_plugin,
-                       global_request_id=context.global_id)
+                       global_request_id=context.global_id,
+                       connect_retries=CONF.neutron.http_retries)
 
     if CONF.neutron.url:
         # TODO(efried): Remove in Rocky
diff --git a/nova/tests/unit/network/test_neutronv2.py b/nova/tests/unit/network/test_neutronv2.py
index d24ac6a31e6e..312b08763a87 100644
--- a/nova/tests/unit/network/test_neutronv2.py
+++ b/nova/tests/unit/network/test_neutronv2.py
@@ -283,6 +283,15 @@ class TestNeutronClient(test.NoDBTestCase):
             exception.Unauthorized,
             client.list_networks)
 
+    def test_neutron_http_retries(self):
+        retries = 42
+        self.flags(http_retries=retries, group='neutron')
+        my_context = context.RequestContext('userid',
+                                            uuids.my_tenant,
+                                            auth_token='token')
+        cl = neutronapi.get_client(my_context)
+        self.assertEqual(retries, cl.httpclient.connect_retries)
+
 
 class TestNeutronv2Base(test.TestCase):
 
diff --git a/releasenotes/notes/neutron-connection-retries-c276010afe238abc.yaml b/releasenotes/notes/neutron-connection-retries-c276010afe238abc.yaml
new file mode 100644
index 000000000000..9a0871f557bf
--- /dev/null
+++ b/releasenotes/notes/neutron-connection-retries-c276010afe238abc.yaml
@@ -0,0 +1,12 @@
+---
+fixes:
+  - |
+    A new config option ``[neutron]http_retries`` is added which defaults to
+    3. It controls how many times to retry a Neutron API call in response to a
+    HTTP connection failure. An example scenario where it will help is when a
+    deployment is using HAProxy and connections get closed after idle time. If
+    an incoming request tries to re-use a connection that is simultaneously
+    being torn down, a HTTP connection failure will occur and previously Nova
+    would fail the entire request. With retries, Nova can be more resilient in
+    this scenario and continue the request if a retry succeeds. Refer to
+    https://launchpad.net/bugs/1866937 for more details.