Merge "Add HTTPS support to K8s API healthchecks"

kuryr_kubernetes/cni/health.py

@@ -11,14 +11,15 @@
 # limitations under the License.
 
 import os
-import requests
 from six.moves import http_client as httplib
 
 from flask import Flask
 from pyroute2 import IPDB
 
 from kuryr.lib._i18n import _
+from kuryr_kubernetes import clients
 from kuryr_kubernetes.cni import utils
+from kuryr_kubernetes import exceptions as exc
 from oslo_config import cfg
 from oslo_log import log as logging
 
@@ -108,7 +109,7 @@ class CNIHealthServer(object):
 
     def readiness_status(self):
         data = 'ok'
-        k8s_conn, k8s_status = self.verify_k8s_connection()
+        k8s_conn = self.verify_k8s_connection()
 
         if not _has_cap(CAP_NET_ADMIN, EFFECTIVE_CAPS):
             error_message = 'NET_ADMIN capabilities not present.'
@@ -117,7 +118,7 @@ class CNIHealthServer(object):
         if not k8s_conn:
             error_message = 'Error when processing k8s healthz request.'
             LOG.error(error_message)
-            return error_message, k8s_status, self.headers
+            return error_message, httplib.INTERNAL_SERVER_ERROR, self.headers
 
         LOG.info('CNI driver readiness verified.')
         return data, httplib.OK, self.headers
@@ -160,8 +161,10 @@ class CNIHealthServer(object):
             raise
 
     def verify_k8s_connection(self):
-        path = '/healthz'
+        k8s = clients.get_kubernetes_client()
-        address = CONF.kubernetes.api_root
+        try:
-        url = address + path
+            k8s.get('/healthz', json=False, headers={'Connection': 'close'})
-        resp = requests.get(url, headers={'Connection': 'close'})
+        except exc.K8sClientException:
-        return resp.content == 'ok', resp.status_code
+            LOG.exception('Exception when trying to reach Kubernetes API.')
+            return False
+        return True

kuryr_kubernetes/controller/managers/health.py

@@ -12,18 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+from six.moves import http_client as httplib
+
 from flask import Flask
+from oslo_config import cfg
+from oslo_log import log as logging
+
 from keystoneauth1 import exceptions as k_exc
 from keystoneclient import client as keystone_client
 from kuryr.lib._i18n import _
 from kuryr.lib import config as kuryr_config
 from kuryr.lib import utils
+from kuryr_kubernetes import clients
+from kuryr_kubernetes import exceptions as exc
 from kuryr_kubernetes.handlers import health as h_health
-import os
-from oslo_config import cfg
-from oslo_log import log as logging
-import requests
-from six.moves import http_client as httplib
 
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
@@ -65,11 +68,11 @@ class HealthServer(object):
                 LOG.error(error_message)
                 return error_message, httplib.NOT_FOUND, self.headers
 
-        k8s_conn, status = self.verify_k8s_connection()
+        k8s_conn = self.verify_k8s_connection()
         if not k8s_conn:
             error_message = 'Error when processing k8s healthz request.'
             LOG.error(error_message)
-            return error_message, status, self.headers
+            return error_message, httplib.INTERNAL_SERVER_ERROR, self.headers
         try:
             self.verify_keystone_connection()
         except k_exc.http.HttpError as h_ex:
@@ -109,11 +112,13 @@ class HealthServer(object):
             raise
 
     def verify_k8s_connection(self):
-        path = '/healthz'
+        k8s = clients.get_kubernetes_client()
-        address = CONF.kubernetes.api_root
+        try:
-        url = address + path
+            k8s.get('/healthz', json=False, headers={'Connection': 'close'})
-        resp = requests.get(url, headers={'Connection': 'close'})
+        except exc.K8sClientException:
-        return resp.content == 'ok', resp.status_code
+            LOG.exception('Exception when trying to reach Kubernetes API.')
+            return False
+        return True
 
     def verify_keystone_connection(self):
         conf_group = kuryr_config.neutron_group.name

kuryr_kubernetes/k8s_client.py

@@ -65,18 +65,21 @@ class K8sClient(object):
             else:
                 self.verify_server = ca_crt_file
 
-    def get(self, path):
+    def get(self, path, json=True, headers=None):
         LOG.debug("Get %(path)s", {'path': path})
         url = self._base_url + path
         header = {}
         if self.token:
             header.update({'Authorization': 'Bearer %s' % self.token})
+        if headers:
+            header.update(headers)
         response = requests.get(url, cert=self.cert,
                                 verify=self.verify_server,
                                 headers=header)
         if not response.ok:
             raise exc.K8sClientException(response.text)
-        return response.json()
+        result = response.json() if json else response.text
+        return result
 
     def _get_url_and_header(self, path):
         url = self._base_url + path

kuryr_kubernetes/tests/unit/cni/test_health.py

@@ -54,9 +54,9 @@ class TestCNIHealthServer(base.TestCase):
                 'verify_k8s_connection')
     def test_readiness_status_k8s_error(self, m_verify_k8s_conn, cap_tester):
         cap_tester.return_value = True
-        m_verify_k8s_conn.return_value = False, 503
+        m_verify_k8s_conn.return_value = False
         resp = self.test_client.get('/ready')
-        self.assertEqual(503, resp.status_code)
+        self.assertEqual(500, resp.status_code)
 
     @mock.patch('pyroute2.IPDB.release')
     def test_liveness_status(self, m_ipdb):

kuryr_kubernetes/tests/unit/controller/managers/test_health.py

@@ -65,11 +65,11 @@ class TestHealthServer(base.TestCase):
     @mock.patch('os.path.exists')
     def test_read_k8s_error(self, m_exist, m_verify_k8s_conn):
         m_exist.return_value = True
-        m_verify_k8s_conn.return_value = False, 503
+        m_verify_k8s_conn.return_value = False
         resp = self.test_client.get('/ready')
 
         m_verify_k8s_conn.assert_called_once()
-        self.assertEqual(503, resp.status_code)
+        self.assertEqual(500, resp.status_code)
 
     @mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
                 'verify_keystone_connection')