Adds CRDs readiness checks to Kuryr-Controller
This patch adds health checks to Kuryr-Controller to ensure that the CRDs that it depends on are defined in the cluster. Partially implements: blueprint k8s-network-policies Change-Id: Ie3b6b6900d7a67402bddcca89b18dea35cc86992
This commit is contained in:
parent
c76497725f
commit
72b67223d8
|
@ -17,6 +17,8 @@ K8S_API_BASE = '/api/v1'
|
|||
K8S_API_NAMESPACES = K8S_API_BASE + '/namespaces'
|
||||
K8S_API_CRD = '/apis/openstack.org/v1'
|
||||
K8S_API_CRD_NAMESPACES = K8S_API_CRD + '/namespaces'
|
||||
K8S_API_CRD_KURYRNETS = K8S_API_CRD + '/kuryrnets'
|
||||
K8S_API_CRD_KURYRNETPOLICIES = K8S_API_CRD + '/kuryrnetpolicies'
|
||||
K8S_API_POLICIES = '/apis/networking.k8s.io/v1/networkpolicies'
|
||||
|
||||
K8S_API_NPWG_CRD = '/apis/k8s.cni.cncf.io/v1'
|
||||
|
|
|
@ -23,6 +23,7 @@ from kuryr.lib._i18n import _
|
|||
from kuryr.lib import config as kuryr_config
|
||||
from kuryr.lib import utils
|
||||
from kuryr_kubernetes import clients
|
||||
from kuryr_kubernetes import constants as k_const
|
||||
from kuryr_kubernetes import exceptions as exc
|
||||
from kuryr_kubernetes.handlers import health as h_health
|
||||
|
||||
|
@ -57,6 +58,16 @@ class HealthServer(object):
|
|||
'/alive', methods=['GET'], view_func=self.liveness_status)
|
||||
self.headers = {'Connection': 'close'}
|
||||
|
||||
def _has_kuryr_crd(self, crd_url):
|
||||
k8s = clients.get_kubernetes_client()
|
||||
try:
|
||||
k8s.get(crd_url, json=False, headers={'Connection': 'close'})
|
||||
except exc.K8sClientException:
|
||||
LOG.exception("Kubernetes Client Exception fetching"
|
||||
" CRD. %s" % exc.K8sClientException)
|
||||
return False
|
||||
return True
|
||||
|
||||
def readiness_status(self):
|
||||
data = 'ok'
|
||||
|
||||
|
@ -85,6 +96,14 @@ class HealthServer(object):
|
|||
LOG.exception(error_message)
|
||||
return error_message, httplib.INTERNAL_SERVER_ERROR, self.headers
|
||||
|
||||
crds = [k_const.K8S_API_CRD_KURYRNETS,
|
||||
k_const.K8S_API_CRD_KURYRNETPOLICIES]
|
||||
for crd in crds:
|
||||
if not self._has_kuryr_crd(crd):
|
||||
error_msg = "Error when processing '%s' CRD request." % crd
|
||||
LOG.error(error_msg)
|
||||
return error_msg, httplib.INTERNAL_SERVER_ERROR, self.headers
|
||||
|
||||
LOG.info('Kuryr Controller readiness verified.')
|
||||
return data, httplib.OK, self.headers
|
||||
|
||||
|
|
|
@ -12,9 +12,12 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from kuryr_kubernetes import constants as k_const
|
||||
from kuryr_kubernetes.controller.managers import health
|
||||
from kuryr_kubernetes import exceptions as k_exc
|
||||
from kuryr_kubernetes.handlers import health as h_health
|
||||
from kuryr_kubernetes.tests import base
|
||||
from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix
|
||||
import mock
|
||||
from oslo_config import cfg as oslo_cfg
|
||||
|
||||
|
@ -32,6 +35,8 @@ class TestHealthServer(base.TestCase):
|
|||
self.srv.application.testing = True
|
||||
self.test_client = self.srv.application.test_client()
|
||||
|
||||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'_has_kuryr_crd')
|
||||
@mock.patch('os.path.exists')
|
||||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'verify_neutron_connection')
|
||||
|
@ -39,20 +44,25 @@ class TestHealthServer(base.TestCase):
|
|||
'verify_keystone_connection')
|
||||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'verify_k8s_connection')
|
||||
def test_read(self, m_verify_k8s_conn, m_verify_keystone_conn,
|
||||
m_verify_neutron_conn, m_exist):
|
||||
def test_readiness(self, m_verify_k8s_conn, m_verify_keystone_conn,
|
||||
m_verify_neutron_conn, m_exist,
|
||||
m_has_kuryr_crd):
|
||||
m_has_kuryr_crd.side_effect = [True, True]
|
||||
m_verify_k8s_conn.return_value = True, 200
|
||||
m_exist.return_value = True
|
||||
|
||||
resp = self.test_client.get('/ready')
|
||||
|
||||
m_verify_k8s_conn.assert_called_once()
|
||||
m_verify_keystone_conn.assert_called_once()
|
||||
m_verify_neutron_conn.assert_called_once_with()
|
||||
self.assertEqual(m_has_kuryr_crd.call_count, 2)
|
||||
|
||||
self.assertEqual(200, resp.status_code)
|
||||
self.assertEqual('ok', resp.data.decode())
|
||||
|
||||
@mock.patch('os.path.exists')
|
||||
def test_read_not_found(self, m_exist):
|
||||
def test_readiness_not_found(self, m_exist):
|
||||
m_exist.return_value = False
|
||||
oslo_cfg.CONF.set_override('vif_pool_driver', 'neutron',
|
||||
group='kubernetes')
|
||||
|
@ -62,7 +72,7 @@ class TestHealthServer(base.TestCase):
|
|||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'verify_k8s_connection')
|
||||
@mock.patch('os.path.exists')
|
||||
def test_read_k8s_error(self, m_exist, m_verify_k8s_conn):
|
||||
def test_readiness_k8s_error(self, m_exist, m_verify_k8s_conn):
|
||||
m_exist.return_value = True
|
||||
m_verify_k8s_conn.return_value = False
|
||||
resp = self.test_client.get('/ready')
|
||||
|
@ -75,7 +85,7 @@ class TestHealthServer(base.TestCase):
|
|||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'verify_k8s_connection')
|
||||
@mock.patch('os.path.exists')
|
||||
def test_read_unauthorized(self, m_exist, m_verify_k8s_conn,
|
||||
def test_readiness_unauthorized(self, m_exist, m_verify_k8s_conn,
|
||||
m_verify_keystone_conn):
|
||||
m_exist.return_value = True
|
||||
m_verify_k8s_conn.return_value = True, 200
|
||||
|
@ -92,8 +102,9 @@ class TestHealthServer(base.TestCase):
|
|||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'verify_k8s_connection')
|
||||
@mock.patch('os.path.exists')
|
||||
def test_read_neutron_error(self, m_exist, m_verify_k8s_conn,
|
||||
m_verify_keystone_conn, m_verify_neutron_conn):
|
||||
def test_readiness_neutron_error(self, m_exist, m_verify_k8s_conn,
|
||||
m_verify_keystone_conn,
|
||||
m_verify_neutron_conn):
|
||||
m_exist.return_value = True
|
||||
m_verify_k8s_conn.return_value = True, 200
|
||||
m_verify_neutron_conn.side_effect = Exception
|
||||
|
@ -102,6 +113,105 @@ class TestHealthServer(base.TestCase):
|
|||
m_verify_neutron_conn.assert_called_once()
|
||||
self.assertEqual(500, resp.status_code)
|
||||
|
||||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'_has_kuryr_crd')
|
||||
@mock.patch('os.path.exists')
|
||||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'verify_neutron_connection')
|
||||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'verify_keystone_connection')
|
||||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'verify_k8s_connection')
|
||||
def test_readiness_kuryrnet_crd_error(self, m_verify_k8s_conn,
|
||||
m_verify_keystone_conn,
|
||||
m_verify_neutron_conn, m_exist,
|
||||
m_has_kuryr_crd):
|
||||
kuryrnets_url = k_const.K8S_API_CRD_KURYRNETS
|
||||
m_has_kuryr_crd.side_effect = [False]
|
||||
|
||||
resp = self.test_client.get('/ready')
|
||||
|
||||
m_has_kuryr_crd.assert_called_with(kuryrnets_url)
|
||||
self.assertEqual(m_has_kuryr_crd.call_count, 1)
|
||||
self.assertEqual(500, resp.status_code)
|
||||
|
||||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'_has_kuryr_crd')
|
||||
@mock.patch('os.path.exists')
|
||||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'verify_neutron_connection')
|
||||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'verify_keystone_connection')
|
||||
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
|
||||
'verify_k8s_connection')
|
||||
def test_readiness_kuryrnetpolicy_crd_error(self, m_verify_k8s_conn,
|
||||
m_verify_keystone_conn,
|
||||
m_verify_neutron_conn, m_exist,
|
||||
m_has_kuryr_crd):
|
||||
kuryrnetpolicies_url = k_const.K8S_API_CRD_KURYRNETPOLICIES
|
||||
m_has_kuryr_crd.side_effect = [True, False]
|
||||
|
||||
resp = self.test_client.get('/ready')
|
||||
|
||||
self.assertEqual(m_has_kuryr_crd.call_count, 2)
|
||||
m_has_kuryr_crd.assert_called_with(kuryrnetpolicies_url)
|
||||
self.assertEqual(500, resp.status_code)
|
||||
|
||||
def test__has_kuryrnet_crd(self):
|
||||
kuryrnet_crd = {
|
||||
"apiVersion": "openstack.org/v1",
|
||||
"items": [
|
||||
|
||||
],
|
||||
"kind": "KuryrNetList",
|
||||
"metadata": {
|
||||
"continue": "",
|
||||
"resourceVersion": "33018",
|
||||
"selfLink": "/apis/openstack.org/v1/kuryrnets"
|
||||
}
|
||||
}
|
||||
|
||||
kubernetes = self.useFixture(k_fix.MockK8sClient()).client
|
||||
kubernetes.get.return_value = kuryrnet_crd
|
||||
|
||||
kuryrnets_url = k_const.K8S_API_CRD_KURYRNETS
|
||||
resp = self.srv._has_kuryr_crd(kuryrnets_url)
|
||||
|
||||
self.assertEqual(resp, True)
|
||||
|
||||
def test__has_kuryrnetpolicy_crd(self):
|
||||
kuryrnetpolicies_crd = {
|
||||
"apiVersion": "openstack.org/v1",
|
||||
"items": [
|
||||
|
||||
],
|
||||
"kind": "KuryrNetPolicyList",
|
||||
"metadata": {
|
||||
"continue": "",
|
||||
"resourceVersion": "34186",
|
||||
"selfLink": "/apis/openstack.org/v1/kuryrnetpolicies"
|
||||
}
|
||||
}
|
||||
kubernetes = self.useFixture(k_fix.MockK8sClient()).client
|
||||
kubernetes.get.return_value = kuryrnetpolicies_crd
|
||||
|
||||
kuryrnetpolicies_url = k_const.K8S_API_CRD_KURYRNETPOLICIES
|
||||
resp = self.srv._has_kuryr_crd(kuryrnetpolicies_url)
|
||||
|
||||
self.assertEqual(resp, True)
|
||||
|
||||
def test__has_kuryr_crd_error(self):
|
||||
crds = [k_const.K8S_API_CRD_KURYRNETS,
|
||||
k_const.K8S_API_CRD_KURYRNETPOLICIES]
|
||||
for crd_url in crds:
|
||||
kubernetes = self.useFixture(k_fix.MockK8sClient()).client
|
||||
kubernetes.get.side_effect = k_exc.K8sClientException
|
||||
|
||||
resp = self.srv._has_kuryr_crd(crd_url)
|
||||
self.assertEqual(resp, False)
|
||||
|
||||
kubernetes.get.assert_called_once()
|
||||
|
||||
@mock.patch.object(_TestHandler, 'is_healthy')
|
||||
def test_liveness(self, m_status):
|
||||
m_status.return_value = True
|
||||
|
|
Loading…
Reference in New Issue