Catch socket.timeout waiting port api

We are facing this bug [1] retries was broken at timeouts catching
timeouts let's us handle retries as expected.

Closes-Bug: #1804649

Change-Id: I8ec6b983dbd1b2a1586b787d05c45d1b9c195074
Story: https://tree.taiga.io/project/tripleo-ci-board/task/403?kanban-status=1447275

tripleoclient/tests/test_utils.py

@@ -20,6 +20,7 @@ import logging
 import mock
 import os
 import os.path
+import socket
 import subprocess
 import tempfile
 
@@ -36,6 +37,8 @@ import yaml
 from tripleoclient import exceptions
 from tripleoclient import utils
 
+from six.moves.urllib import error as url_error
+
 
 class TestRunAnsiblePlaybook(TestCase):
     def setUp(self):
@@ -1161,3 +1164,58 @@ class TestDeploymentPythonInterpreter(TestCase):
         args.deployment_python_interpreter = 'foo'
         py = utils.get_deployment_python_interpreter(args)
         self.assertEqual(py, 'foo')
+
+
+class TestWaitApiPortReady(TestCase):
+    @mock.patch('six.moves.urllib.request.urlopen')
+    def test_success(self, urlopen_mock):
+        has_errors = utils.wait_api_port_ready(8080)
+        self.assertFalse(has_errors)
+
+    @mock.patch(
+        'six.moves.urllib.request.urlopen',
+        side_effect=[
+            url_error.HTTPError("", 201, None, None, None), socket.timeout,
+            url_error.URLError("")
+        ] * 10)
+    @mock.patch('time.sleep')
+    def test_throw_exception_at_max_retries(self, urlopen_mock, sleep_mock):
+        with self.assertRaises(RuntimeError):
+            utils.wait_api_port_ready(8080)
+        self.assertEqual(urlopen_mock.call_count, 30)
+        self.assertEqual(sleep_mock.call_count, 30)
+
+    @mock.patch(
+        'six.moves.urllib.request.urlopen',
+        side_effect=[
+            socket.timeout,
+            url_error.URLError(""),
+            url_error.HTTPError("", 201, None, None, None), None
+        ])
+    @mock.patch('time.sleep')
+    def test_recovers_from_exception(self, urlopen_mock, sleep_mock):
+        self.assertFalse(utils.wait_api_port_ready(8080))
+        self.assertEqual(urlopen_mock.call_count, 4)
+        self.assertEqual(sleep_mock.call_count, 4)
+
+    @mock.patch(
+        'six.moves.urllib.request.urlopen',
+        side_effect=[
+            socket.timeout,
+            url_error.URLError(""),
+            url_error.HTTPError("", 300, None, None, None)
+        ] * 10)
+    @mock.patch('time.sleep')
+    def test_recovers_from_multiple_choices_error_code(self, urlopen_mock,
+                                                       sleep_mock):
+        self.assertTrue(utils.wait_api_port_ready(8080))
+        self.assertEqual(urlopen_mock.call_count, 3)
+        self.assertEqual(sleep_mock.call_count, 3)
+
+    @mock.patch('six.moves.urllib.request.urlopen', side_effect=NameError)
+    @mock.patch('time.sleep')
+    def test_dont_retry_at_unknown_exception(self, urlopen_mock, sleep_mock):
+        with self.assertRaises(NameError):
+            utils.wait_api_port_ready(8080)
+        self.assertEqual(urlopen_mock.call_count, 1)
+        self.assertEqual(sleep_mock.call_count, 1)

tripleoclient/utils.py

@@ -1078,19 +1078,30 @@ def wait_api_port_ready(api_port, host='127.0.0.1'):
 
     :return boolean
     """
+    log = logging.getLogger(__name__ + ".wait_api_port_ready")
+    urlopen_timeout = 1
+    max_retries = 30
     count = 0
-    while count < 30:
+    while count < max_retries:
         time.sleep(1)
         count += 1
         try:
-            request.urlopen("http://%s:%s/" % (host, api_port), timeout=1)
+            request.urlopen(
+                "http://%s:%s/" % (host, api_port), timeout=urlopen_timeout)
+            return False
         except url_error.HTTPError as he:
             if he.code == 300:
                 return True
             pass
         except url_error.URLError:
             pass
-    return False
+        except socket.timeout:
+            log.warning(
+                "Timeout at attempt {} of {} after {}s waiting for API port..."
+                .format(count, max_retries, urlopen_timeout))
+            pass
+    raise RuntimeError(
+        "wait_api_port_ready: Max retries {} reached".format(max_retries))
 
 
 def bulk_symlink(log, src, dst, tmpd='/tmp'):