Add timeout for get_endpoint_certificate

This commit works around the lack of a proper timeout in the
ssl.get_server_certificate() call. A proper timeout is available in
python 3.8+ (see https://bugs.python.org/issue31870).

We work around the lack of timeout by first attempting a basic socket
connection to the (host, port) with a short timeout (default: 10s). If
that is successful we can continue to fetch the server certificate;
otherwise we abandon the attempt.

Test Plan / Failure Path:

PASS: Verify proper timeout when subcloud is offline/shutdown
PASS: Verify proper behaviour when subcloud is available

Regression:
PASS: Verify feature logging

Story: 2008960
Task: 43545

Change-Id: If0f7c926e136fc776dbc74ed27b767af1af2cf80
Signed-off-by: Kyle MacLeod <kyle.macleod@windriver.com>

sysinv/sysinv/sysinv/sysinv/cert_mon/certificate_mon_manager.py

@@ -66,6 +66,9 @@ cert_mon_opts = [
                help='Size of subcloud audit greenpool.'
                     'Set to 0 to disable use of greenpool '
                     '(force serial audit).'),
+    cfg.IntOpt('certificate_timeout_secs',
+               default=10,
+               help='Connection timeout for certificate check (in seconds)'),
 ]
 
 CONF = cfg.CONF
@@ -211,7 +214,9 @@ class CertificateMonManager(periodic_task.PeriodicTasks):
         try:
             subcloud_sysinv_url = utils.dc_get_subcloud_sysinv_url(
                 subcloud_name, my_dc_token())
-            sc_ssl_cert = utils.get_endpoint_certificate(subcloud_sysinv_url)
+            sc_ssl_cert = utils.get_endpoint_certificate(
+                subcloud_sysinv_url,
+                timeout_secs=CONF.certmon.certificate_timeout_secs)
 
         except Exception:
             if not utils.is_subcloud_online(subcloud_name, my_dc_token()):

sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py

@@ -21,6 +21,7 @@ import json
 import os
 import re
 import ssl
+import socket
 import tempfile
 
 import requests
@@ -525,10 +526,27 @@ def get_sc_intermediate_ca_secret(sc):
     return kube_op.kube_get_secret(secret_name, CERT_NAMESPACE_SYS_CONTROLLER)
 
 
-def get_endpoint_certificate(endpoint):
+def get_endpoint_certificate(endpoint, timeout_secs=10):
     url = urlparse(endpoint)
     host = url.hostname
     port = url.port
+    if timeout_secs is not None and timeout_secs > 0:
+        # The call to ssl.get_server_certificate blocks for a long time if the
+        # server is not available. A timeout is not available in python 2.7.
+        # See https://bugs.python.org/issue31870
+        # Until the timeout=<val> option is available in
+        # get_server_certificate(), we first check if the port is open
+        # by connecting using a timeout, then we do the certificate check:
+        sock = None
+        try:
+            sock = socket.create_connection((host, port), timeout=timeout_secs)
+        except Exception:
+            LOG.warn("get_endpoint_certificate: connection failed to %s:%s",
+                     host, port)
+            raise
+        finally:
+            if sock is not None:
+                sock.close()
     return ssl.get_server_certificate((host, port))