Add HTTPS support to K8s API healthchecks

Our Controller and CNI healthchecks are checking the connection to the
K8s API. This is done using the requests library directly and not
through K8sClient wrapper. Because of that we've forgot to add HTTPS
support there (DevStack runs K8s with HTTP).

This commit fix that by making /healthz requests to go through
K8sClient wrapper.

Change-Id: I7d2657dfb123a7641eb838d7ddecde769bbb4318
Closes-Bug: 1759293

kuryr_kubernetes/cni/health.py

@@ -11,14 +11,15 @@
 # limitations under the License.
 
 import os
-import requests
 from six.moves import http_client as httplib
 
 from flask import Flask
 from pyroute2 import IPDB
 
 from kuryr.lib._i18n import _
+from kuryr_kubernetes import clients
 from kuryr_kubernetes.cni import utils
+from kuryr_kubernetes import exceptions as exc
 from oslo_config import cfg
 from oslo_log import log as logging
 
@@ -108,7 +109,7 @@ class CNIHealthServer(object):
 
     def readiness_status(self):
         data = 'ok'
-        k8s_conn, k8s_status = self.verify_k8s_connection()
+        k8s_conn = self.verify_k8s_connection()
 
         if not _has_cap(CAP_NET_ADMIN, EFFECTIVE_CAPS):
             error_message = 'NET_ADMIN capabilities not present.'
@@ -117,7 +118,7 @@ class CNIHealthServer(object):
         if not k8s_conn:
             error_message = 'Error when processing k8s healthz request.'
             LOG.error(error_message)
-            return error_message, k8s_status, self.headers
+            return error_message, httplib.INTERNAL_SERVER_ERROR, self.headers
 
         LOG.info('CNI driver readiness verified.')
         return data, httplib.OK, self.headers
@@ -160,8 +161,10 @@ class CNIHealthServer(object):
             raise
 
     def verify_k8s_connection(self):
-        path = '/healthz'
-        address = CONF.kubernetes.api_root
-        url = address + path
-        resp = requests.get(url, headers={'Connection': 'close'})
-        return resp.content == 'ok', resp.status_code
+        k8s = clients.get_kubernetes_client()
+        try:
+            k8s.get('/healthz', json=False, headers={'Connection': 'close'})
+        except exc.K8sClientException:
+            LOG.exception('Exception when trying to reach Kubernetes API.')
+            return False
+        return True

kuryr_kubernetes/controller/managers/health.py

@@ -12,18 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+from six.moves import http_client as httplib
+
 from flask import Flask
+from oslo_config import cfg
+from oslo_log import log as logging
+
 from keystoneauth1 import exceptions as k_exc
 from keystoneclient import client as keystone_client
 from kuryr.lib._i18n import _
 from kuryr.lib import config as kuryr_config
 from kuryr.lib import utils
+from kuryr_kubernetes import clients
+from kuryr_kubernetes import exceptions as exc
 from kuryr_kubernetes.handlers import health as h_health
-import os
-from oslo_config import cfg
-from oslo_log import log as logging
-import requests
-from six.moves import http_client as httplib
 
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
@@ -65,11 +68,11 @@ class HealthServer(object):
                 LOG.error(error_message)
                 return error_message, httplib.NOT_FOUND, self.headers
 
-        k8s_conn, status = self.verify_k8s_connection()
+        k8s_conn = self.verify_k8s_connection()
         if not k8s_conn:
             error_message = 'Error when processing k8s healthz request.'
             LOG.error(error_message)
-            return error_message, status, self.headers
+            return error_message, httplib.INTERNAL_SERVER_ERROR, self.headers
         try:
             self.verify_keystone_connection()
         except k_exc.http.HttpError as h_ex:
@@ -109,11 +112,13 @@ class HealthServer(object):
             raise
 
     def verify_k8s_connection(self):
-        path = '/healthz'
-        address = CONF.kubernetes.api_root
-        url = address + path
-        resp = requests.get(url, headers={'Connection': 'close'})
-        return resp.content == 'ok', resp.status_code
+        k8s = clients.get_kubernetes_client()
+        try:
+            k8s.get('/healthz', json=False, headers={'Connection': 'close'})
+        except exc.K8sClientException:
+            LOG.exception('Exception when trying to reach Kubernetes API.')
+            return False
+        return True
 
     def verify_keystone_connection(self):
         conf_group = kuryr_config.neutron_group.name

kuryr_kubernetes/k8s_client.py

@@ -65,18 +65,21 @@ class K8sClient(object):
             else:
                 self.verify_server = ca_crt_file
 
-    def get(self, path):
+    def get(self, path, json=True, headers=None):
         LOG.debug("Get %(path)s", {'path': path})
         url = self._base_url + path
         header = {}
         if self.token:
             header.update({'Authorization': 'Bearer %s' % self.token})
+        if headers:
+            header.update(headers)
         response = requests.get(url, cert=self.cert,
                                 verify=self.verify_server,
                                 headers=header)
         if not response.ok:
             raise exc.K8sClientException(response.text)
-        return response.json()
+        result = response.json() if json else response.text
+        return result
 
     def _get_url_and_header(self, path):
         url = self._base_url + path

kuryr_kubernetes/tests/unit/cni/test_health.py

@@ -54,9 +54,9 @@ class TestCNIHealthServer(base.TestCase):
                 'verify_k8s_connection')
     def test_readiness_status_k8s_error(self, m_verify_k8s_conn, cap_tester):
         cap_tester.return_value = True
-        m_verify_k8s_conn.return_value = False, 503
+        m_verify_k8s_conn.return_value = False
         resp = self.test_client.get('/ready')
-        self.assertEqual(503, resp.status_code)
+        self.assertEqual(500, resp.status_code)
 
     @mock.patch('pyroute2.IPDB.release')
     def test_liveness_status(self, m_ipdb):

kuryr_kubernetes/tests/unit/controller/managers/test_health.py

@@ -65,11 +65,11 @@ class TestHealthServer(base.TestCase):
     @mock.patch('os.path.exists')
     def test_read_k8s_error(self, m_exist, m_verify_k8s_conn):
         m_exist.return_value = True
-        m_verify_k8s_conn.return_value = False, 503
+        m_verify_k8s_conn.return_value = False
         resp = self.test_client.get('/ready')
 
         m_verify_k8s_conn.assert_called_once()
-        self.assertEqual(503, resp.status_code)
+        self.assertEqual(500, resp.status_code)
 
     @mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
                 'verify_keystone_connection')