From d224999cb0368a2b54e29372b90f92e93b43c170 Mon Sep 17 00:00:00 2001 From: Feilong Wang Date: Fri, 9 Aug 2019 15:23:30 +1200 Subject: [PATCH] Improve log of k8s health status check When the k8s cluster is created in a private network without FIP and without a LB FIP, the cluster is not reachable by Magnum control plane. For this case, the cluster health status is unknown for Magnum and Magnum should not connect the cluster to avoid unnecessary logs. Task: 36187 Story: 2006383 Change-Id: Ibb589ec51f91f05a334c907560ef5fe57cbfbffb --- magnum/drivers/common/k8s_monitor.py | 15 ++++++++++++++- magnum/objects/fields.py | 3 ++- magnum/tests/unit/conductor/test_monitors.py | 16 ++++++++++++++-- magnum/tests/unit/objects/test_objects.py | 2 +- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/magnum/drivers/common/k8s_monitor.py b/magnum/drivers/common/k8s_monitor.py index 7739569345..1e8cb9c285 100644 --- a/magnum/drivers/common/k8s_monitor.py +++ b/magnum/drivers/common/k8s_monitor.py @@ -50,10 +50,23 @@ class K8sMonitor(monitors.MonitorBase): def poll_health_status(self): k8s_api = k8s.create_k8s_api(self.context, self.cluster) - status, reason = self._poll_health_status(k8s_api) + if self._is_cluster_accessible(): + status, reason = self._poll_health_status(k8s_api) + else: + status = m_fields.ClusterHealthStatus.UNKNOWN + reason = {"api": "The cluster %s is not accessible." % + self.cluster.name} + self.data['health_status'] = status self.data['health_status_reason'] = reason + def _is_cluster_accessible(self): + lb_fip = self.cluster.labels.get("master_lb_floating_ip_enabled") + lb_fip_enabled = strutils.bool_from_string(lb_fip) + return (self.cluster.floating_ip_enabled or + (lb_fip_enabled and + self.cluster.cluster_template.master_lb_enabled)) + def _compute_res_util(self, res): res_total = 0 for node in self.data['nodes']: diff --git a/magnum/objects/fields.py b/magnum/objects/fields.py index 6bb88b994d..64786f8afc 100644 --- a/magnum/objects/fields.py +++ b/magnum/objects/fields.py @@ -52,8 +52,9 @@ class ClusterStatus(fields.Enum): class ClusterHealthStatus(fields.Enum): HEALTHY = 'HEALTHY' UNHEALTHY = 'UNHEALTHY' + UNKNOWN = 'UNKNOWN' - ALL = (HEALTHY, UNHEALTHY) + ALL = (HEALTHY, UNHEALTHY, UNKNOWN) STATUS_FAILED = (UNHEALTHY) diff --git a/magnum/tests/unit/conductor/test_monitors.py b/magnum/tests/unit/conductor/test_monitors.py index 9e44272b49..73f5013f04 100644 --- a/magnum/tests/unit/conductor/test_monitors.py +++ b/magnum/tests/unit/conductor/test_monitors.py @@ -47,10 +47,10 @@ class MonitorsTestCase(base.TestCase): def setUp(self): super(MonitorsTestCase, self).setUp() - cluster = utils.get_test_cluster(node_addresses=['1.2.3.4'], api_address='https://5.6.7.8:2376', - master_addresses=['10.0.0.6']) + master_addresses=['10.0.0.6'], + labels={}) self.cluster = objects.Cluster(self.context, **cluster) nodegroups = utils.get_nodegroups_for_cluster( node_addresses=['1.2.3.4'], master_addresses=['10.0.0.6']) @@ -511,3 +511,15 @@ class MonitorsTestCase(base.TestCase): self.assertEqual(self.k8s_monitor.data['health_status_reason'], {'api': 'ok', 'k8s-cluster-node-0.Ready': False, 'api': 'ok', 'k8s-cluster-node-1.Ready': True}) + + @mock.patch('magnum.conductor.k8s_api.create_k8s_api') + def test_k8s_monitor_health_unreachable_cluster(self, mock_k8s_api): + mock_nodes = mock.MagicMock() + mock_node = mock.MagicMock() + mock_node.status = mock.MagicMock() + mock_nodes.items = [mock_node] + self.k8s_monitor.cluster.floating_ip_enabled = False + + self.k8s_monitor.poll_health_status() + self.assertEqual(self.k8s_monitor.data['health_status'], + m_fields.ClusterHealthStatus.UNKNOWN) diff --git a/magnum/tests/unit/objects/test_objects.py b/magnum/tests/unit/objects/test_objects.py index ec565133a8..5aa0eceed1 100644 --- a/magnum/tests/unit/objects/test_objects.py +++ b/magnum/tests/unit/objects/test_objects.py @@ -355,7 +355,7 @@ class TestObject(test_base.TestCase, _TestObject): # For more information on object version testing, read # https://docs.openstack.org/magnum/latest/contributor/objects.html object_data = { - 'Cluster': '1.21-11ab13dfd5cb53578d398008d758dab8', + 'Cluster': '1.21-e28dc2dd599433a6249f7901ebf7b162', 'ClusterTemplate': '1.19-3b0b2b3933d0955abf3ab40111744960', 'Certificate': '1.1-1924dc077daa844f0f9076332ef96815', 'MyObj': '1.0-34c4b1aadefd177b13f9a2f894cc23cd',