Fix a possible receive timeout

This patch fixes a possible receive timeout caused by a slow response from the
driver agent. For example if the database is very slow.

Closes-Bug: #2032890

Change-Id: I9079030a5fef9dc44da242adab3c568666777451
(cherry picked from commit 2a84a218ef)
(cherry picked from commit f1142a99a9)
(cherry picked from commit 5ec74a4dbe)
(cherry picked from commit 3f764c5d2b)
(cherry picked from commit 450c3b9f47)

octavia_lib/api/drivers/driver_lib.py

@@ -58,11 +58,19 @@ class DriverLibrary():
 
     def _recv(self, sock):
         size_str = b''
-        char = sock.recv(1)
         begin = time.time()
-        while char != b'\n':
-            size_str += char
-            char = sock.recv(1)
+        while True:
+            try:
+                char = sock.recv(1)
+            except socket.timeout:
+                # We could have an overloaded DB and the query may take too
+                # long, so as long as DRIVER_AGENT_TIMEOUT hasn't expired,
+                # let's keep trying while not blocking everything.
+                pass
+            else:
+                if char == b'\n':
+                    break
+                size_str += char
             if time.time() - begin > DRIVER_AGENT_TIMEOUT:
                 raise driver_exceptions.DriverAgentTimeout(
                     fault_string=('The driver agent did not respond in {} '

octavia_lib/tests/unit/api/drivers/test_driver_lib.py

@@ -11,6 +11,7 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import socket
 from unittest import mock
 
 from octavia_lib.api.drivers import driver_lib
@@ -45,7 +46,7 @@ class TestDriverLib(base.TestCase):
     @mock.patch('builtins.memoryview')
     def test_recv(self, mock_memoryview):
         mock_socket = mock.MagicMock()
-        mock_socket.recv.side_effect = [b'1', b'\n', b'2', b'\n', b'3', b'\n']
+        mock_socket.recv.side_effect = [b'1', b'\n', b'2', b'3', b'\n']
         mock_socket.recv_into.return_value = 1
         mv_mock = mock.MagicMock()
         mock_memoryview.return_value = mv_mock
@@ -71,6 +72,25 @@ class TestDriverLib(base.TestCase):
             self.assertRaises(driver_exceptions.DriverAgentTimeout,
                               self.driver_lib._recv, mock_socket)
 
+    @mock.patch('builtins.memoryview')
+    def test_recv_with_timeout(self, mock_memoryview):
+        mock_socket = mock.MagicMock()
+        mock_socket.recv.side_effect = [socket.timeout, b'1', b'\n', b'2',
+                                        b'3', b'\n']
+        mock_socket.recv_into.return_value = 1
+        mv_mock = mock.MagicMock()
+        mock_memoryview.return_value = mv_mock
+        mv_mock.tobytes.return_value = b'"test data"'
+
+        response = self.driver_lib._recv(mock_socket)
+
+        calls = [mock.call(1), mock.call(1)]
+
+        mock_socket.recv.assert_has_calls(calls)
+        mock_socket.recv_into.assert_called_once_with(
+            mv_mock.__getitem__(), 1)
+        self.assertEqual('test data', response)
+
     @mock.patch('octavia_lib.api.drivers.driver_lib.DriverLibrary._recv')
     def test_send(self, mock_recv):
         mock_socket = mock.MagicMock()

releasenotes/notes/Fix-possible-receive-timeout-error-f6d2454c48eb10cf.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixed a possible receive timeout if the driver agent didn't respond inside
+    five seconds. For example if you have a very slow database.