From 75c8450b1b1b9bee55a4f9815d56138aed537436 Mon Sep 17 00:00:00 2001 From: Duc Truong Date: Wed, 9 Sep 2020 00:20:27 +0000 Subject: [PATCH] Add hypervisor status polling health check Add new health check type to health policy called HYPERVISOR_STATUS_POLLING. When set, it considers a node to be unhealthy if the hypervisor on which the node is running has a disabled status or down state. Bump health policy version to 1.2 with the addition of HYPERVISOR_STATUS_POLLING. Change-Id: Ie558fb6ef7bf06cf4065fec5498e229d3146b0b2 --- senlin/common/consts.py | 2 + senlin/drivers/os/nova_v2.py | 35 ++++++ senlin/engine/health_manager.py | 37 +++++- senlin/engine/node.py | 4 +- senlin/policies/health_policy.py | 10 +- senlin/profiles/base.py | 4 +- senlin/profiles/os/nova/server.py | 104 ++++++++++++++-- senlin/tests/unit/drivers/test_nova_v2.py | 60 ++++++++++ .../tests/unit/engine/test_health_manager.py | 98 +++++++++++++++ senlin/tests/unit/engine/test_node.py | 7 +- .../tests/unit/policies/test_health_policy.py | 15 ++- .../tests/unit/profiles/test_nova_server.py | 113 +++++++++++++++++- setup.cfg | 1 + 13 files changed, 458 insertions(+), 32 deletions(-) diff --git a/senlin/common/consts.py b/senlin/common/consts.py index 052cfa1d6..e31091005 100755 --- a/senlin/common/consts.py +++ b/senlin/common/consts.py @@ -284,9 +284,11 @@ EVENT_LEVELS = { DETECTION_TYPES = ( LIFECYCLE_EVENTS, NODE_STATUS_POLLING, NODE_STATUS_POLL_URL, + HYPERVISOR_STATUS_POLLING, # LB_STATUS_POLLING, ) = ( 'LIFECYCLE_EVENTS', 'NODE_STATUS_POLLING', 'NODE_STATUS_POLL_URL', + 'HYPERVISOR_STATUS_POLLING', # 'LB_STATUS_POLLING', ) diff --git a/senlin/drivers/os/nova_v2.py b/senlin/drivers/os/nova_v2.py index 029c0d25b..f6d5031f1 100644 --- a/senlin/drivers/os/nova_v2.py +++ b/senlin/drivers/os/nova_v2.py @@ -265,6 +265,41 @@ class NovaClient(base.DriverBase): def hypervisor_get(self, hypervisor): return self.conn.compute.get_hypervisor(hypervisor) + @sdk.translate_exception + def hypervisor_find(self, name_or_id, ignore_missing=False): + # try finding hypervisor by id + try: + return self.conn.compute.get_hypervisor(name_or_id) + except sdk_exc.HttpException: + # ignore http exception and instead get list and check by name + pass + + # if the hypervisor could not be found using id, search list using name + results = self.conn.compute.hypervisors( + hypervisor_hostname_pattern=name_or_id) + + result = None + for maybe_result in results: + name_value = maybe_result.name + + if name_value == name_or_id: + # Only allow one resource to be found. If we already + # found a match, raise an exception to show it. + if result is None: + result = maybe_result + else: + msg = "More than one hypervisor exists with the name '%s'." + msg = (msg % name_or_id) + raise sdk_exc.DuplicateResource(msg) + + if result is not None: + return result + + if ignore_missing: + return None + raise sdk_exc.ResourceNotFound( + "No hypervisor found for %s" % (name_or_id)) + @sdk.translate_exception def service_list(self): return self.conn.compute.services() diff --git a/senlin/engine/health_manager.py b/senlin/engine/health_manager.py index 9606cff32..1cd3cf36d 100644 --- a/senlin/engine/health_manager.py +++ b/senlin/engine/health_manager.py @@ -105,6 +105,9 @@ class HealthCheckType(object): elif detection_type == consts.NODE_STATUS_POLL_URL: return NodePollUrlHealthCheck( cid, interval, node_update_timeout, detection_params[0]) + elif detection_type == consts.HYPERVISOR_STATUS_POLLING: + return HypervisorPollStatusHealthCheck( + cid, interval, node_update_timeout, detection_params[0]) else: raise Exception( 'Invalid detection type: {}'.format(detection_type)) @@ -171,7 +174,36 @@ class NodePollStatusHealthCheck(HealthCheckType): # Return False to mark the node as unhealthy if we are outside the # grace period. - return (entity.do_healthcheck(ctx) or + return (entity.do_healthcheck(ctx, consts.NODE_STATUS_POLLING) or + self._node_within_grace_period(node)) + except Exception as ex: + LOG.warning( + 'Error when performing health check on node %s: %s', + node.id, ex + ) + + # treat node as healthy when an exception is encountered + return True + + +class HypervisorPollStatusHealthCheck(HealthCheckType): + def run_health_check(self, ctx, node): + """Routine to be executed for polling hypervisor status. + + :returns: True if node is healthy. False otherwise. + """ + try: + # create engine node from db node + entity = node_mod.Node._from_object(ctx, node) + + # If health check returns True, return True to mark node as + # healthy. Else return True to mark node as healthy if we are still + # within the node's grace period to allow the node to warm-up. + # Return False to mark the node as unhealthy if we are outside the + # grace period. + + return (entity.do_healthcheck(ctx, + consts.HYPERVISOR_STATUS_POLLING) or self._node_within_grace_period(node)) except Exception as ex: LOG.warning( @@ -313,7 +345,8 @@ class HealthCheck(object): def get_health_check_types(self): polling_types = [consts.NODE_STATUS_POLLING, - consts.NODE_STATUS_POLL_URL] + consts.NODE_STATUS_POLL_URL, + consts.HYPERVISOR_STATUS_POLLING] detection_types = self.check_type.split(',') if all(check in polling_types for check in detection_types): diff --git a/senlin/engine/node.py b/senlin/engine/node.py index 9c9f4a020..fd7de4021 100644 --- a/senlin/engine/node.py +++ b/senlin/engine/node.py @@ -349,7 +349,7 @@ class Node(object): return True - def do_healthcheck(self, context): + def do_healthcheck(self, context, health_check_type): """health check a node. This function is supposed to be invoked from the health manager to @@ -358,7 +358,7 @@ class Node(object): :returns: True if node is healthy. False otherwise. """ - return pb.Profile.healthcheck_object(context, self) + return pb.Profile.healthcheck_object(context, self, health_check_type) def do_recover(self, context, action): """recover a node. diff --git a/senlin/policies/health_policy.py b/senlin/policies/health_policy.py index cfa234144..cdb041b13 100644 --- a/senlin/policies/health_policy.py +++ b/senlin/policies/health_policy.py @@ -29,7 +29,7 @@ LOG = logging.getLogger(__name__) class HealthPolicy(base.Policy): """Policy for health management of a cluster.""" - VERSION = '1.1' + VERSION = '1.2' VERSIONS = { '1.0': [ {'status': consts.EXPERIMENTAL, 'since': '2017.02'}, @@ -38,6 +38,9 @@ class HealthPolicy(base.Policy): '1.1': [ {'status': consts.SUPPORTED, 'since': '2018.09'} ], + '1.2': [ + {'status': consts.SUPPORTED, 'since': '2020.09'} + ], } PRIORITY = 600 @@ -107,7 +110,7 @@ class HealthPolicy(base.Policy): DETECTION_INTERVAL: schema.Integer( _("Number of seconds between pollings. Only " "required when type is 'NODE_STATUS_POLLING' or " - "'NODE_STATUS_POLL_URL'."), + "'NODE_STATUS_POLL_URL' or 'HYPERVISOR_STATUS_POLLING."), default=60, ), NODE_UPDATE_TIMEOUT: schema.Integer( @@ -316,7 +319,8 @@ class HealthPolicy(base.Policy): # check valid detection types polling_types = [consts.NODE_STATUS_POLLING, - consts.NODE_STATUS_POLL_URL] + consts.NODE_STATUS_POLL_URL, + consts.HYPERVISOR_STATUS_POLLING] has_valid_polling_types = all( d.type in polling_types diff --git a/senlin/profiles/base.py b/senlin/profiles/base.py index 7f370ef82..e5bc0fa1b 100644 --- a/senlin/profiles/base.py +++ b/senlin/profiles/base.py @@ -305,9 +305,9 @@ class Profile(object): @classmethod @profiler.trace('Profile.check_object', hide_args=False) - def healthcheck_object(cls, ctx, obj): + def healthcheck_object(cls, ctx, obj, health_check_type): profile = cls.load(ctx, profile_id=obj.profile_id) - return profile.do_healthcheck(obj) + return profile.do_healthcheck(obj, health_check_type) @classmethod @profiler.trace('Profile.recover_object', hide_args=False) diff --git a/senlin/profiles/os/nova/server.py b/senlin/profiles/os/nova/server.py index a826c68e8..2fea13e91 100644 --- a/senlin/profiles/os/nova/server.py +++ b/senlin/profiles/os/nova/server.py @@ -1638,22 +1638,20 @@ class ServerProfile(base.Profile): return True - def do_healthcheck(self, obj): + def do_healthcheck(self, obj, health_check_type): """Healthcheck operation. - This method checks if a server node is healthy by getting the server - status from nova. A server is considered unhealthy if it does not - exist or its status is one of the following: - - ERROR - - SHUTOFF - - DELETED + This method checks if a node is healthy. If health check type is + NODE_STATUS_POLLING it will check the server status. If health check + type is HYPERVISOR_STATUS_POLLING it will check the hypervisor state + and status. :param obj: The node object to operate on. + :param health_check_type: The type of health check. Either + NODE_STATUS_POLLING or HYPERVISOR_STATUS_POLLING. :return status: True indicates node is healthy, False indicates it is unhealthy. """ - unhealthy_server_status = [consts.VS_ERROR, consts.VS_SHUTOFF, - consts.VS_DELETED] if not obj.physical_id: if obj.status == 'BUILD' or obj.status == 'CREATING': @@ -1687,6 +1685,34 @@ class ServerProfile(base.Profile): consts.POLL_STATUS_PASS, obj.name) return True + if health_check_type == consts.NODE_STATUS_POLLING: + return self._do_healthcheck_server(obj, server) + elif health_check_type == consts.HYPERVISOR_STATUS_POLLING: + return self._do_healthcheck_hypervisor(obj, server) + else: + LOG.info('%s for %s: ignoring invalid health check type %s', + consts.POLL_STATUS_PASS, obj.name, health_check_type) + return True + + def _do_healthcheck_server(self, obj, server): + """Healthcheck operation based on server. + + This method checks if a server node is healthy by getting the server + status from nova. A server is considered unhealthy if it does not + exist or its status is one of the following: + - ERROR + - SHUTOFF + - DELETED + + :param obj: The node object to operate on. + :param server: The server object associated with the node. + :return status: True indicates node is healthy, False indicates + it is unhealthy. + """ + + unhealthy_server_status = [consts.VS_ERROR, consts.VS_SHUTOFF, + consts.VS_DELETED] + if server.status in unhealthy_server_status: LOG.info('%s for %s: server status is unhealthy.', consts.POLL_STATUS_FAIL, obj.name) @@ -1695,6 +1721,66 @@ class ServerProfile(base.Profile): LOG.info('%s for %s', consts.POLL_STATUS_PASS, obj.name) return True + def _do_healthcheck_hypervisor(self, obj, server): + """Healthcheck operation based on hypervisor. + + This method checks if a server node is healthy by getting the + hypervisor state and status from nova. A server is considered + unhealthy if the hypervisor it is running on has a state that is down + or a status that is disabled. + + :param obj: The node object to operate on. + :param server: The server object associated with the node. + :return status: True indicates node is healthy, False indicates + it is unhealthy. + """ + + if server.hypervisor_hostname != "": + try: + hv = self.compute(obj).hypervisor_find( + server.hypervisor_hostname) + except Exception as ex: + if isinstance(ex, exc.InternalError) and ex.code == 404: + # treat resource not found exception as unhealthy + LOG.info('%s for %s: hypervisor %s was not found.', + consts.POLL_STATUS_FAIL, obj.name, + server.hypervisor_hostname) + return False + else: + # treat all other exceptions as healthy + LOG.info( + '%s for %s: Exception when trying to get hypervisor ' + 'info for %s, but ignoring this error: %s.', + consts.POLL_STATUS_PASS, obj.name, + server.hypervisor_hostname, ex.message) + return True + + if hv is None: + # no hypervisor information is available, treat the node as + # healthy + LOG.info( + '%s for %s: No hypervisor information was returned but ' + 'ignoring this error.', + consts.POLL_STATUS_PASS, obj.name) + return True + + if hv.state == 'down': + LOG.info('%s for %s: server status is unhealthy because ' + 'hypervisor %s state is down', + consts.POLL_STATUS_FAIL, obj.name, + server.hypervisor_hostname) + return False + + if hv.status == 'disabled': + LOG.info('%s for %s: server status is unhealthy because ' + 'hypervisor %s status is disabled', + consts.POLL_STATUS_FAIL, obj.name, + server.hypervisor_hostname) + return False + + LOG.info('%s for %s', consts.POLL_STATUS_PASS, obj.name) + return True + def do_recover(self, obj, **options): """Handler for recover operation. diff --git a/senlin/tests/unit/drivers/test_nova_v2.py b/senlin/tests/unit/drivers/test_nova_v2.py index e91e2dc9f..b7d5df838 100644 --- a/senlin/tests/unit/drivers/test_nova_v2.py +++ b/senlin/tests/unit/drivers/test_nova_v2.py @@ -15,6 +15,7 @@ from unittest import mock from oslo_config import cfg +from senlin.common import exception as exc from senlin.drivers.os import nova_v2 from senlin.drivers import sdk from senlin.tests.unit.common import base @@ -552,6 +553,65 @@ class TestNovaV2(base.SenlinTestCase): d.hypervisor_get('k') self.compute.get_hypervisor.assert_called_once_with('k') + def test_hypervisor_find(self): + d = nova_v2.NovaClient(self.conn_params) + self.compute.get_hypervisor.return_value = mock.Mock() + + d.hypervisor_find('k') + + self.compute.get_hypervisor.assert_called_once_with('k') + + def test_hypervisor_find_name(self): + d = nova_v2.NovaClient(self.conn_params) + self.compute.get_hypervisor.side_effect = sdk_exc.HttpException + fake_result = mock.Mock() + fake_result.name = 'FAKE_HV' + self.compute.hypervisors.return_value = [mock.Mock(name='not_it'), + fake_result] + + r = d.hypervisor_find('FAKE_HV') + + self.compute.get_hypervisor.assert_called_once_with('FAKE_HV') + self.compute.hypervisors.assert_called_once_with( + hypervisor_hostname_pattern='FAKE_HV') + self.assertEqual(r, fake_result) + + def test_hypervisor_find_name_duplicate(self): + d = nova_v2.NovaClient(self.conn_params) + self.compute.get_hypervisor.side_effect = sdk_exc.HttpException + fake_result = mock.Mock() + fake_result.name = 'FAKE_HV' + self.compute.hypervisors.return_value = [fake_result, fake_result] + + self.assertRaises(exc.InternalError, d.hypervisor_find, 'FAKE_HV') + + self.compute.get_hypervisor.assert_called_once_with('FAKE_HV') + self.compute.hypervisors.assert_called_once_with( + hypervisor_hostname_pattern='FAKE_HV') + + def test_hypervisor_find_name_ignore_missing(self): + d = nova_v2.NovaClient(self.conn_params) + self.compute.get_hypervisor.side_effect = sdk_exc.HttpException + self.compute.hypervisors.return_value = [mock.Mock(name='not_it')] + + r = d.hypervisor_find('FAKE_HV', True) + + self.compute.get_hypervisor.assert_called_once_with('FAKE_HV') + self.compute.hypervisors.assert_called_once_with( + hypervisor_hostname_pattern='FAKE_HV') + self.assertIsNone(r) + + def test_hypervisor_find_name_not_found(self): + d = nova_v2.NovaClient(self.conn_params) + self.compute.get_hypervisor.side_effect = sdk_exc.HttpException + self.compute.hypervisors.return_value = [mock.Mock(name='not_it')] + + self.assertRaises(exc.InternalError, d.hypervisor_find, 'FAKE_HV') + + self.compute.get_hypervisor.assert_called_once_with('FAKE_HV') + self.compute.hypervisors.assert_called_once_with( + hypervisor_hostname_pattern='FAKE_HV') + def test_service_list(self): d = nova_v2.NovaClient(self.conn_params) d.service_list() diff --git a/senlin/tests/unit/engine/test_health_manager.py b/senlin/tests/unit/engine/test_health_manager.py index adc6cc248..783277454 100644 --- a/senlin/tests/unit/engine/test_health_manager.py +++ b/senlin/tests/unit/engine/test_health_manager.py @@ -326,6 +326,15 @@ class TestHealthCheckType(base.SenlinTestCase): 'poll_url_retry_limit': '', 'poll_url_retry_interval': '' }, + { + 'type': 'HYPERVISOR_STATUS_POLLING', + 'poll_url': '', + 'poll_url_ssl_verify': True, + 'poll_url_conn_error_as_unhealthy': True, + 'poll_url_healthy_response': '', + 'poll_url_retry_limit': '', + 'poll_url_retry_interval': '' + }, { 'type': 'NODE_STATUS_POLL_URL', 'poll_url': '', @@ -492,6 +501,95 @@ class TestNodePollStatusHealthCheck(base.SenlinTestCase): mock_tu.assert_called_once_with(node.updated_at, 1) +class TestHypervisorPollStatusHealthCheck(base.SenlinTestCase): + def setUp(self): + super(TestHypervisorPollStatusHealthCheck, self).setUp() + + self.hc = hm.HypervisorPollStatusHealthCheck( + cluster_id='CLUSTER_ID', + interval=1, node_update_timeout=1, params='' + ) + + @mock.patch.object(node_mod.Node, '_from_object') + @mock.patch.object(tu, 'is_older_than') + def test_run_health_check_healthy(self, mock_tu, mock_node_obj): + x_entity = mock.Mock() + x_entity.do_healthcheck.return_value = True + mock_node_obj.return_value = x_entity + + ctx = mock.Mock() + node = mock.Mock(id='FAKE_NODE1', status="ERROR", + updated_at='2018-08-13 18:00:00', + init_at='2018-08-13 17:00:00') + + # do it + res = self.hc.run_health_check(ctx, node) + + self.assertTrue(res) + mock_tu.assert_not_called() + + @mock.patch.object(node_mod.Node, '_from_object') + @mock.patch.object(tu, 'is_older_than') + def test_run_health_check_healthy_internal_error( + self, mock_tu, mock_node_obj): + x_entity = mock.Mock() + x_entity.do_healthcheck.side_effect = exc.InternalError( + message='error') + mock_node_obj.return_value = x_entity + + ctx = mock.Mock() + node = mock.Mock(id='FAKE_NODE1', status="ERROR", + updated_at='2018-08-13 18:00:00', + init_at='2018-08-13 17:00:00') + + # do it + res = self.hc.run_health_check(ctx, node) + + self.assertTrue(res) + mock_tu.assert_not_called() + + @mock.patch.object(node_mod.Node, '_from_object') + @mock.patch.object(tu, 'is_older_than') + def test_run_health_check_unhealthy(self, mock_tu, mock_node_obj): + x_entity = mock.Mock() + x_entity.do_healthcheck.return_value = False + mock_node_obj.return_value = x_entity + + mock_tu.return_value = True + + ctx = mock.Mock() + node = mock.Mock(id='FAKE_NODE1', status="ERROR", + updated_at='2018-08-13 18:00:00', + init_at='2018-08-13 17:00:00') + + # do it + res = self.hc.run_health_check(ctx, node) + + self.assertFalse(res) + mock_tu.assert_called_once_with(node.updated_at, 1) + + @mock.patch.object(node_mod.Node, '_from_object') + @mock.patch.object(tu, 'is_older_than') + def test_run_health_check_unhealthy_within_timeout( + self, mock_tu, mock_node_obj): + x_entity = mock.Mock() + x_entity.do_healthcheck.return_value = False + mock_node_obj.return_value = x_entity + + mock_tu.return_value = False + + ctx = mock.Mock() + node = mock.Mock(id='FAKE_NODE1', status="ERROR", + updated_at='2018-08-13 18:00:00', + init_at='2018-08-13 17:00:00') + + # do it + res = self.hc.run_health_check(ctx, node) + + self.assertTrue(res) + mock_tu.assert_called_once_with(node.updated_at, 1) + + class TestNodePollUrlHealthCheck(base.SenlinTestCase): def setUp(self): super(TestNodePollUrlHealthCheck, self).setUp() diff --git a/senlin/tests/unit/engine/test_node.py b/senlin/tests/unit/engine/test_node.py index b765bf84d..4cc752b14 100644 --- a/senlin/tests/unit/engine/test_node.py +++ b/senlin/tests/unit/engine/test_node.py @@ -600,15 +600,16 @@ class TestNode(base.SenlinTestCase): node.status = consts.NS_ACTIVE node.physical_id = 'd94d6333-82e6-4f87-b7ab-b786776df9d1' mock_healthcheck.return_value = True - res = node.do_healthcheck(self.context) + res = node.do_healthcheck(self.context, consts.NODE_STATUS_POLLING) self.assertTrue(res) - mock_healthcheck.assert_called_once_with(self.context, node) + mock_healthcheck.assert_called_once_with(self.context, node, + consts.NODE_STATUS_POLLING) def test_node_healthcheck_no_physical_id(self): node = nodem.Node('node1', PROFILE_ID, '') - res = node.do_healthcheck(self.context) + res = node.do_healthcheck(self.context, consts.NODE_STATUS_POLLING) self.assertFalse(res) diff --git a/senlin/tests/unit/policies/test_health_policy.py b/senlin/tests/unit/policies/test_health_policy.py index e4976aa2f..8173ce0dc 100644 --- a/senlin/tests/unit/policies/test_health_policy.py +++ b/senlin/tests/unit/policies/test_health_policy.py @@ -35,7 +35,7 @@ class TestHealthPolicy(base.SenlinTestCase): self.spec = { 'type': 'senlin.policy.health', - 'version': '1.1', + 'version': '1.2', 'properties': { 'detection': { "detection_modes": [ @@ -82,7 +82,7 @@ class TestHealthPolicy(base.SenlinTestCase): spec = { 'type': 'senlin.policy.health', - 'version': '1.1', + 'version': '1.2', 'properties': { 'detection': { "detection_modes": [ @@ -105,7 +105,7 @@ class TestHealthPolicy(base.SenlinTestCase): self.assertIsNone(hp.id) self.assertEqual('test-policy', hp.name) - self.assertEqual('senlin.policy.health-1.1', hp.type) + self.assertEqual('senlin.policy.health-1.2', hp.type) self.assertEqual(detection_modes, hp.detection_modes) self.assertEqual(60, hp.interval) self.assertEqual([{'name': 'REBUILD', 'params': None}], @@ -114,13 +114,16 @@ class TestHealthPolicy(base.SenlinTestCase): def test_policy_init_ops(self): spec = { 'type': 'senlin.policy.health', - 'version': '1.1', + 'version': '1.2', 'properties': { 'detection': { "detection_modes": [ { 'type': 'NODE_STATUS_POLLING' }, + { + 'type': 'HYPERVISOR_STATUS_POLLING' + }, { 'type': 'NODE_STATUS_POLL_URL' }, @@ -148,7 +151,7 @@ class TestHealthPolicy(base.SenlinTestCase): # check result self.assertIsNone(hp.id) self.assertEqual('test-policy', hp.name) - self.assertEqual('senlin.policy.health-1.1', hp.type) + self.assertEqual('senlin.policy.health-1.2', hp.type) self.assertEqual(60, hp.interval) self.assertEqual([{'name': 'REBUILD', 'params': None}], hp.recover_actions) @@ -216,7 +219,7 @@ class TestHealthPolicy(base.SenlinTestCase): 'node_force_recreate': False, 'recovery_conditional': 'ANY_FAILED' }, - 'version': '1.1' + 'version': '1.2' } } diff --git a/senlin/tests/unit/profiles/test_nova_server.py b/senlin/tests/unit/profiles/test_nova_server.py index 9741ee0b8..1728bbf78 100644 --- a/senlin/tests/unit/profiles/test_nova_server.py +++ b/senlin/tests/unit/profiles/test_nova_server.py @@ -16,6 +16,7 @@ from unittest import mock from oslo_config import cfg from oslo_utils import encodeutils +from senlin.common import consts from senlin.common import exception as exc from senlin.objects import node as node_ob from senlin.profiles import base as profiles_base @@ -1715,7 +1716,7 @@ class TestNovaServerBasic(base.SenlinTestCase): test_server = mock.Mock(physical_id='FAKE_ID') - res = profile.do_healthcheck(test_server) + res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING) cc.server_get.assert_called_once_with('FAKE_ID') self.assertTrue(res) @@ -1728,7 +1729,7 @@ class TestNovaServerBasic(base.SenlinTestCase): test_server = mock.Mock(physical_id='FAKE_ID') - res = profile.do_healthcheck(test_server) + res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING) cc.server_get.assert_called_once_with('FAKE_ID') self.assertTrue(res) @@ -1742,7 +1743,7 @@ class TestNovaServerBasic(base.SenlinTestCase): test_server = mock.Mock(physical_id='FAKE_ID') - res = profile.do_healthcheck(test_server) + res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING) cc.server_get.assert_called_once_with('FAKE_ID') self.assertTrue(res) @@ -1756,7 +1757,7 @@ class TestNovaServerBasic(base.SenlinTestCase): test_server = mock.Mock(physical_id='FAKE_ID') - res = profile.do_healthcheck(test_server) + res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING) cc.server_get.assert_called_once_with('FAKE_ID') self.assertFalse(res) @@ -1771,11 +1772,113 @@ class TestNovaServerBasic(base.SenlinTestCase): test_server = mock.Mock(physical_id='FAKE_ID') - res = profile.do_healthcheck(test_server) + res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING) cc.server_get.assert_called_once_with('FAKE_ID') self.assertFalse(res) + def test_do_healthcheck_empty_hv_name(self): + profile = server.ServerProfile('t', self.spec) + + cc = mock.Mock() + cc.hypervisor_find.return_value = None + cc.server_get.return_value = mock.Mock(hypervisor_hostname='') + profile._computeclient = cc + + test_server = mock.Mock(physical_id='FAKE_ID') + + res = profile.do_healthcheck(test_server, + consts.HYPERVISOR_STATUS_POLLING) + cc.server_get.assert_called_once_with('FAKE_ID') + cc.hypervisor_find.assert_not_called() + self.assertTrue(res) + + def test_do_healthcheck_empty_hv_obj(self): + profile = server.ServerProfile('t', self.spec) + + cc = mock.Mock() + cc.hypervisor_find.return_value = None + cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV') + profile._computeclient = cc + + test_server = mock.Mock(physical_id='FAKE_ID') + + res = profile.do_healthcheck(test_server, + consts.HYPERVISOR_STATUS_POLLING) + cc.server_get.assert_called_once_with('FAKE_ID') + cc.hypervisor_find.assert_called_once_with('FAKE_HV') + self.assertTrue(res) + + def test_do_healthcheck_hv_exception(self): + profile = server.ServerProfile('t', self.spec) + + cc = mock.Mock() + cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV') + ex = exc.InternalError(code=503, message='Error') + cc.hypervisor_find.side_effect = ex + profile._computeclient = cc + + test_server = mock.Mock(physical_id='FAKE_ID') + + res = profile.do_healthcheck(test_server, + consts.HYPERVISOR_STATUS_POLLING) + + cc.server_get.assert_called_once_with('FAKE_ID') + cc.hypervisor_find.assert_called_once_with('FAKE_HV') + self.assertTrue(res) + + def test_do_healthcheck_hv_not_found(self): + profile = server.ServerProfile('t', self.spec) + + cc = mock.Mock() + cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV') + ex = exc.InternalError(code=404, message='No Hypervisor found') + cc.hypervisor_find.side_effect = ex + profile._computeclient = cc + + test_server = mock.Mock(physical_id='FAKE_ID') + + res = profile.do_healthcheck(test_server, + consts.HYPERVISOR_STATUS_POLLING) + + cc.server_get.assert_called_once_with('FAKE_ID') + cc.hypervisor_find.assert_called_once_with('FAKE_HV') + self.assertFalse(res) + + def test_do_healthcheck_hv_down(self): + profile = server.ServerProfile('t', self.spec) + + cc = mock.Mock() + cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV') + cc.hypervisor_find.return_value = mock.Mock(state='down') + profile._computeclient = cc + + test_server = mock.Mock(physical_id='FAKE_ID') + + res = profile.do_healthcheck(test_server, + consts.HYPERVISOR_STATUS_POLLING) + + cc.server_get.assert_called_once_with('FAKE_ID') + cc.hypervisor_find.assert_called_once_with('FAKE_HV') + self.assertFalse(res) + + def test_do_healthcheck_hv_disabled(self): + profile = server.ServerProfile('t', self.spec) + + cc = mock.Mock() + cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV') + cc.hypervisor_find.return_value = mock.Mock(status='disabled') + profile._computeclient = cc + + test_server = mock.Mock(physical_id='FAKE_ID') + + res = profile.do_healthcheck(test_server, + consts.HYPERVISOR_STATUS_POLLING) + + cc.server_get.assert_called_once_with('FAKE_ID') + cc.hypervisor_find.assert_called_once_with('FAKE_HV') + self.assertFalse(res) + @mock.patch.object(server.ServerProfile, 'do_delete') @mock.patch.object(server.ServerProfile, 'do_create') def test_do_recover_operation_is_none(self, mock_create, mock_delete): diff --git a/setup.cfg b/setup.cfg index 4fb0cf893..3295a7d68 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,6 +62,7 @@ senlin.policies = senlin.policy.scaling-1.0 = senlin.policies.scaling_policy:ScalingPolicy senlin.policy.health-1.0 = senlin.policies.health_policy:HealthPolicy senlin.policy.health-1.1 = senlin.policies.health_policy:HealthPolicy + senlin.policy.health-1.2 = senlin.policies.health_policy:HealthPolicy senlin.policy.loadbalance-1.0 = senlin.policies.lb_policy:LoadBalancingPolicy senlin.policy.loadbalance-1.1 = senlin.policies.lb_policy:LoadBalancingPolicy senlin.policy.loadbalance-1.2 = senlin.policies.lb_policy:LoadBalancingPolicy