Merge "Add hypervisor status polling health check"
This commit is contained in:
commit
5238a34c7d
|
@ -284,9 +284,11 @@ EVENT_LEVELS = {
|
|||
|
||||
DETECTION_TYPES = (
|
||||
LIFECYCLE_EVENTS, NODE_STATUS_POLLING, NODE_STATUS_POLL_URL,
|
||||
HYPERVISOR_STATUS_POLLING,
|
||||
# LB_STATUS_POLLING,
|
||||
) = (
|
||||
'LIFECYCLE_EVENTS', 'NODE_STATUS_POLLING', 'NODE_STATUS_POLL_URL',
|
||||
'HYPERVISOR_STATUS_POLLING',
|
||||
# 'LB_STATUS_POLLING',
|
||||
)
|
||||
|
||||
|
|
|
@ -265,6 +265,41 @@ class NovaClient(base.DriverBase):
|
|||
def hypervisor_get(self, hypervisor):
|
||||
return self.conn.compute.get_hypervisor(hypervisor)
|
||||
|
||||
@sdk.translate_exception
|
||||
def hypervisor_find(self, name_or_id, ignore_missing=False):
|
||||
# try finding hypervisor by id
|
||||
try:
|
||||
return self.conn.compute.get_hypervisor(name_or_id)
|
||||
except sdk_exc.HttpException:
|
||||
# ignore http exception and instead get list and check by name
|
||||
pass
|
||||
|
||||
# if the hypervisor could not be found using id, search list using name
|
||||
results = self.conn.compute.hypervisors(
|
||||
hypervisor_hostname_pattern=name_or_id)
|
||||
|
||||
result = None
|
||||
for maybe_result in results:
|
||||
name_value = maybe_result.name
|
||||
|
||||
if name_value == name_or_id:
|
||||
# Only allow one resource to be found. If we already
|
||||
# found a match, raise an exception to show it.
|
||||
if result is None:
|
||||
result = maybe_result
|
||||
else:
|
||||
msg = "More than one hypervisor exists with the name '%s'."
|
||||
msg = (msg % name_or_id)
|
||||
raise sdk_exc.DuplicateResource(msg)
|
||||
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
if ignore_missing:
|
||||
return None
|
||||
raise sdk_exc.ResourceNotFound(
|
||||
"No hypervisor found for %s" % (name_or_id))
|
||||
|
||||
@sdk.translate_exception
|
||||
def service_list(self):
|
||||
return self.conn.compute.services()
|
||||
|
|
|
@ -105,6 +105,9 @@ class HealthCheckType(object):
|
|||
elif detection_type == consts.NODE_STATUS_POLL_URL:
|
||||
return NodePollUrlHealthCheck(
|
||||
cid, interval, node_update_timeout, detection_params[0])
|
||||
elif detection_type == consts.HYPERVISOR_STATUS_POLLING:
|
||||
return HypervisorPollStatusHealthCheck(
|
||||
cid, interval, node_update_timeout, detection_params[0])
|
||||
else:
|
||||
raise Exception(
|
||||
'Invalid detection type: {}'.format(detection_type))
|
||||
|
@ -171,7 +174,36 @@ class NodePollStatusHealthCheck(HealthCheckType):
|
|||
# Return False to mark the node as unhealthy if we are outside the
|
||||
# grace period.
|
||||
|
||||
return (entity.do_healthcheck(ctx) or
|
||||
return (entity.do_healthcheck(ctx, consts.NODE_STATUS_POLLING) or
|
||||
self._node_within_grace_period(node))
|
||||
except Exception as ex:
|
||||
LOG.warning(
|
||||
'Error when performing health check on node %s: %s',
|
||||
node.id, ex
|
||||
)
|
||||
|
||||
# treat node as healthy when an exception is encountered
|
||||
return True
|
||||
|
||||
|
||||
class HypervisorPollStatusHealthCheck(HealthCheckType):
|
||||
def run_health_check(self, ctx, node):
|
||||
"""Routine to be executed for polling hypervisor status.
|
||||
|
||||
:returns: True if node is healthy. False otherwise.
|
||||
"""
|
||||
try:
|
||||
# create engine node from db node
|
||||
entity = node_mod.Node._from_object(ctx, node)
|
||||
|
||||
# If health check returns True, return True to mark node as
|
||||
# healthy. Else return True to mark node as healthy if we are still
|
||||
# within the node's grace period to allow the node to warm-up.
|
||||
# Return False to mark the node as unhealthy if we are outside the
|
||||
# grace period.
|
||||
|
||||
return (entity.do_healthcheck(ctx,
|
||||
consts.HYPERVISOR_STATUS_POLLING) or
|
||||
self._node_within_grace_period(node))
|
||||
except Exception as ex:
|
||||
LOG.warning(
|
||||
|
@ -313,7 +345,8 @@ class HealthCheck(object):
|
|||
|
||||
def get_health_check_types(self):
|
||||
polling_types = [consts.NODE_STATUS_POLLING,
|
||||
consts.NODE_STATUS_POLL_URL]
|
||||
consts.NODE_STATUS_POLL_URL,
|
||||
consts.HYPERVISOR_STATUS_POLLING]
|
||||
|
||||
detection_types = self.check_type.split(',')
|
||||
if all(check in polling_types for check in detection_types):
|
||||
|
|
|
@ -349,7 +349,7 @@ class Node(object):
|
|||
|
||||
return True
|
||||
|
||||
def do_healthcheck(self, context):
|
||||
def do_healthcheck(self, context, health_check_type):
|
||||
"""health check a node.
|
||||
|
||||
This function is supposed to be invoked from the health manager to
|
||||
|
@ -358,7 +358,7 @@ class Node(object):
|
|||
:returns: True if node is healthy. False otherwise.
|
||||
"""
|
||||
|
||||
return pb.Profile.healthcheck_object(context, self)
|
||||
return pb.Profile.healthcheck_object(context, self, health_check_type)
|
||||
|
||||
def do_recover(self, context, action):
|
||||
"""recover a node.
|
||||
|
|
|
@ -29,7 +29,7 @@ LOG = logging.getLogger(__name__)
|
|||
class HealthPolicy(base.Policy):
|
||||
"""Policy for health management of a cluster."""
|
||||
|
||||
VERSION = '1.1'
|
||||
VERSION = '1.2'
|
||||
VERSIONS = {
|
||||
'1.0': [
|
||||
{'status': consts.EXPERIMENTAL, 'since': '2017.02'},
|
||||
|
@ -38,6 +38,9 @@ class HealthPolicy(base.Policy):
|
|||
'1.1': [
|
||||
{'status': consts.SUPPORTED, 'since': '2018.09'}
|
||||
],
|
||||
'1.2': [
|
||||
{'status': consts.SUPPORTED, 'since': '2020.09'}
|
||||
],
|
||||
}
|
||||
PRIORITY = 600
|
||||
|
||||
|
@ -113,7 +116,7 @@ class HealthPolicy(base.Policy):
|
|||
DETECTION_INTERVAL: schema.Integer(
|
||||
_("Number of seconds between pollings. Only "
|
||||
"required when type is 'NODE_STATUS_POLLING' or "
|
||||
"'NODE_STATUS_POLL_URL'."),
|
||||
"'NODE_STATUS_POLL_URL' or 'HYPERVISOR_STATUS_POLLING."),
|
||||
default=60,
|
||||
),
|
||||
NODE_UPDATE_TIMEOUT: schema.Integer(
|
||||
|
@ -322,7 +325,8 @@ class HealthPolicy(base.Policy):
|
|||
|
||||
# check valid detection types
|
||||
polling_types = [consts.NODE_STATUS_POLLING,
|
||||
consts.NODE_STATUS_POLL_URL]
|
||||
consts.NODE_STATUS_POLL_URL,
|
||||
consts.HYPERVISOR_STATUS_POLLING]
|
||||
|
||||
has_valid_polling_types = all(
|
||||
d.type in polling_types
|
||||
|
|
|
@ -305,9 +305,9 @@ class Profile(object):
|
|||
|
||||
@classmethod
|
||||
@profiler.trace('Profile.check_object', hide_args=False)
|
||||
def healthcheck_object(cls, ctx, obj):
|
||||
def healthcheck_object(cls, ctx, obj, health_check_type):
|
||||
profile = cls.load(ctx, profile_id=obj.profile_id)
|
||||
return profile.do_healthcheck(obj)
|
||||
return profile.do_healthcheck(obj, health_check_type)
|
||||
|
||||
@classmethod
|
||||
@profiler.trace('Profile.recover_object', hide_args=False)
|
||||
|
|
|
@ -1772,22 +1772,20 @@ class ServerProfile(base.Profile):
|
|||
|
||||
return True
|
||||
|
||||
def do_healthcheck(self, obj):
|
||||
def do_healthcheck(self, obj, health_check_type):
|
||||
"""Healthcheck operation.
|
||||
|
||||
This method checks if a server node is healthy by getting the server
|
||||
status from nova. A server is considered unhealthy if it does not
|
||||
exist or its status is one of the following:
|
||||
- ERROR
|
||||
- SHUTOFF
|
||||
- DELETED
|
||||
This method checks if a node is healthy. If health check type is
|
||||
NODE_STATUS_POLLING it will check the server status. If health check
|
||||
type is HYPERVISOR_STATUS_POLLING it will check the hypervisor state
|
||||
and status.
|
||||
|
||||
:param obj: The node object to operate on.
|
||||
:param health_check_type: The type of health check. Either
|
||||
NODE_STATUS_POLLING or HYPERVISOR_STATUS_POLLING.
|
||||
:return status: True indicates node is healthy, False indicates
|
||||
it is unhealthy.
|
||||
"""
|
||||
unhealthy_server_status = [consts.VS_ERROR, consts.VS_SHUTOFF,
|
||||
consts.VS_DELETED]
|
||||
|
||||
if not obj.physical_id:
|
||||
if obj.status == 'BUILD' or obj.status == 'CREATING':
|
||||
|
@ -1821,6 +1819,34 @@ class ServerProfile(base.Profile):
|
|||
consts.POLL_STATUS_PASS, obj.name)
|
||||
return True
|
||||
|
||||
if health_check_type == consts.NODE_STATUS_POLLING:
|
||||
return self._do_healthcheck_server(obj, server)
|
||||
elif health_check_type == consts.HYPERVISOR_STATUS_POLLING:
|
||||
return self._do_healthcheck_hypervisor(obj, server)
|
||||
else:
|
||||
LOG.info('%s for %s: ignoring invalid health check type %s',
|
||||
consts.POLL_STATUS_PASS, obj.name, health_check_type)
|
||||
return True
|
||||
|
||||
def _do_healthcheck_server(self, obj, server):
|
||||
"""Healthcheck operation based on server.
|
||||
|
||||
This method checks if a server node is healthy by getting the server
|
||||
status from nova. A server is considered unhealthy if it does not
|
||||
exist or its status is one of the following:
|
||||
- ERROR
|
||||
- SHUTOFF
|
||||
- DELETED
|
||||
|
||||
:param obj: The node object to operate on.
|
||||
:param server: The server object associated with the node.
|
||||
:return status: True indicates node is healthy, False indicates
|
||||
it is unhealthy.
|
||||
"""
|
||||
|
||||
unhealthy_server_status = [consts.VS_ERROR, consts.VS_SHUTOFF,
|
||||
consts.VS_DELETED]
|
||||
|
||||
if server.status in unhealthy_server_status:
|
||||
LOG.info('%s for %s: server status is unhealthy.',
|
||||
consts.POLL_STATUS_FAIL, obj.name)
|
||||
|
@ -1829,6 +1855,66 @@ class ServerProfile(base.Profile):
|
|||
LOG.info('%s for %s', consts.POLL_STATUS_PASS, obj.name)
|
||||
return True
|
||||
|
||||
def _do_healthcheck_hypervisor(self, obj, server):
|
||||
"""Healthcheck operation based on hypervisor.
|
||||
|
||||
This method checks if a server node is healthy by getting the
|
||||
hypervisor state and status from nova. A server is considered
|
||||
unhealthy if the hypervisor it is running on has a state that is down
|
||||
or a status that is disabled.
|
||||
|
||||
:param obj: The node object to operate on.
|
||||
:param server: The server object associated with the node.
|
||||
:return status: True indicates node is healthy, False indicates
|
||||
it is unhealthy.
|
||||
"""
|
||||
|
||||
if server.hypervisor_hostname != "":
|
||||
try:
|
||||
hv = self.compute(obj).hypervisor_find(
|
||||
server.hypervisor_hostname)
|
||||
except Exception as ex:
|
||||
if isinstance(ex, exc.InternalError) and ex.code == 404:
|
||||
# treat resource not found exception as unhealthy
|
||||
LOG.info('%s for %s: hypervisor %s was not found.',
|
||||
consts.POLL_STATUS_FAIL, obj.name,
|
||||
server.hypervisor_hostname)
|
||||
return False
|
||||
else:
|
||||
# treat all other exceptions as healthy
|
||||
LOG.info(
|
||||
'%s for %s: Exception when trying to get hypervisor '
|
||||
'info for %s, but ignoring this error: %s.',
|
||||
consts.POLL_STATUS_PASS, obj.name,
|
||||
server.hypervisor_hostname, ex.message)
|
||||
return True
|
||||
|
||||
if hv is None:
|
||||
# no hypervisor information is available, treat the node as
|
||||
# healthy
|
||||
LOG.info(
|
||||
'%s for %s: No hypervisor information was returned but '
|
||||
'ignoring this error.',
|
||||
consts.POLL_STATUS_PASS, obj.name)
|
||||
return True
|
||||
|
||||
if hv.state == 'down':
|
||||
LOG.info('%s for %s: server status is unhealthy because '
|
||||
'hypervisor %s state is down',
|
||||
consts.POLL_STATUS_FAIL, obj.name,
|
||||
server.hypervisor_hostname)
|
||||
return False
|
||||
|
||||
if hv.status == 'disabled':
|
||||
LOG.info('%s for %s: server status is unhealthy because '
|
||||
'hypervisor %s status is disabled',
|
||||
consts.POLL_STATUS_FAIL, obj.name,
|
||||
server.hypervisor_hostname)
|
||||
return False
|
||||
|
||||
LOG.info('%s for %s', consts.POLL_STATUS_PASS, obj.name)
|
||||
return True
|
||||
|
||||
def do_recover(self, obj, **options):
|
||||
"""Handler for recover operation.
|
||||
|
||||
|
|
|
@ -15,6 +15,7 @@ from unittest import mock
|
|||
|
||||
from oslo_config import cfg
|
||||
|
||||
from senlin.common import exception as exc
|
||||
from senlin.drivers.os import nova_v2
|
||||
from senlin.drivers import sdk
|
||||
from senlin.tests.unit.common import base
|
||||
|
@ -552,6 +553,65 @@ class TestNovaV2(base.SenlinTestCase):
|
|||
d.hypervisor_get('k')
|
||||
self.compute.get_hypervisor.assert_called_once_with('k')
|
||||
|
||||
def test_hypervisor_find(self):
|
||||
d = nova_v2.NovaClient(self.conn_params)
|
||||
self.compute.get_hypervisor.return_value = mock.Mock()
|
||||
|
||||
d.hypervisor_find('k')
|
||||
|
||||
self.compute.get_hypervisor.assert_called_once_with('k')
|
||||
|
||||
def test_hypervisor_find_name(self):
|
||||
d = nova_v2.NovaClient(self.conn_params)
|
||||
self.compute.get_hypervisor.side_effect = sdk_exc.HttpException
|
||||
fake_result = mock.Mock()
|
||||
fake_result.name = 'FAKE_HV'
|
||||
self.compute.hypervisors.return_value = [mock.Mock(name='not_it'),
|
||||
fake_result]
|
||||
|
||||
r = d.hypervisor_find('FAKE_HV')
|
||||
|
||||
self.compute.get_hypervisor.assert_called_once_with('FAKE_HV')
|
||||
self.compute.hypervisors.assert_called_once_with(
|
||||
hypervisor_hostname_pattern='FAKE_HV')
|
||||
self.assertEqual(r, fake_result)
|
||||
|
||||
def test_hypervisor_find_name_duplicate(self):
|
||||
d = nova_v2.NovaClient(self.conn_params)
|
||||
self.compute.get_hypervisor.side_effect = sdk_exc.HttpException
|
||||
fake_result = mock.Mock()
|
||||
fake_result.name = 'FAKE_HV'
|
||||
self.compute.hypervisors.return_value = [fake_result, fake_result]
|
||||
|
||||
self.assertRaises(exc.InternalError, d.hypervisor_find, 'FAKE_HV')
|
||||
|
||||
self.compute.get_hypervisor.assert_called_once_with('FAKE_HV')
|
||||
self.compute.hypervisors.assert_called_once_with(
|
||||
hypervisor_hostname_pattern='FAKE_HV')
|
||||
|
||||
def test_hypervisor_find_name_ignore_missing(self):
|
||||
d = nova_v2.NovaClient(self.conn_params)
|
||||
self.compute.get_hypervisor.side_effect = sdk_exc.HttpException
|
||||
self.compute.hypervisors.return_value = [mock.Mock(name='not_it')]
|
||||
|
||||
r = d.hypervisor_find('FAKE_HV', True)
|
||||
|
||||
self.compute.get_hypervisor.assert_called_once_with('FAKE_HV')
|
||||
self.compute.hypervisors.assert_called_once_with(
|
||||
hypervisor_hostname_pattern='FAKE_HV')
|
||||
self.assertIsNone(r)
|
||||
|
||||
def test_hypervisor_find_name_not_found(self):
|
||||
d = nova_v2.NovaClient(self.conn_params)
|
||||
self.compute.get_hypervisor.side_effect = sdk_exc.HttpException
|
||||
self.compute.hypervisors.return_value = [mock.Mock(name='not_it')]
|
||||
|
||||
self.assertRaises(exc.InternalError, d.hypervisor_find, 'FAKE_HV')
|
||||
|
||||
self.compute.get_hypervisor.assert_called_once_with('FAKE_HV')
|
||||
self.compute.hypervisors.assert_called_once_with(
|
||||
hypervisor_hostname_pattern='FAKE_HV')
|
||||
|
||||
def test_service_list(self):
|
||||
d = nova_v2.NovaClient(self.conn_params)
|
||||
d.service_list()
|
||||
|
|
|
@ -326,6 +326,15 @@ class TestHealthCheckType(base.SenlinTestCase):
|
|||
'poll_url_retry_limit': '',
|
||||
'poll_url_retry_interval': ''
|
||||
},
|
||||
{
|
||||
'type': 'HYPERVISOR_STATUS_POLLING',
|
||||
'poll_url': '',
|
||||
'poll_url_ssl_verify': True,
|
||||
'poll_url_conn_error_as_unhealthy': True,
|
||||
'poll_url_healthy_response': '',
|
||||
'poll_url_retry_limit': '',
|
||||
'poll_url_retry_interval': ''
|
||||
},
|
||||
{
|
||||
'type': 'NODE_STATUS_POLL_URL',
|
||||
'poll_url': '',
|
||||
|
@ -492,6 +501,95 @@ class TestNodePollStatusHealthCheck(base.SenlinTestCase):
|
|||
mock_tu.assert_called_once_with(node.updated_at, 1)
|
||||
|
||||
|
||||
class TestHypervisorPollStatusHealthCheck(base.SenlinTestCase):
|
||||
def setUp(self):
|
||||
super(TestHypervisorPollStatusHealthCheck, self).setUp()
|
||||
|
||||
self.hc = hm.HypervisorPollStatusHealthCheck(
|
||||
cluster_id='CLUSTER_ID',
|
||||
interval=1, node_update_timeout=1, params=''
|
||||
)
|
||||
|
||||
@mock.patch.object(node_mod.Node, '_from_object')
|
||||
@mock.patch.object(tu, 'is_older_than')
|
||||
def test_run_health_check_healthy(self, mock_tu, mock_node_obj):
|
||||
x_entity = mock.Mock()
|
||||
x_entity.do_healthcheck.return_value = True
|
||||
mock_node_obj.return_value = x_entity
|
||||
|
||||
ctx = mock.Mock()
|
||||
node = mock.Mock(id='FAKE_NODE1', status="ERROR",
|
||||
updated_at='2018-08-13 18:00:00',
|
||||
init_at='2018-08-13 17:00:00')
|
||||
|
||||
# do it
|
||||
res = self.hc.run_health_check(ctx, node)
|
||||
|
||||
self.assertTrue(res)
|
||||
mock_tu.assert_not_called()
|
||||
|
||||
@mock.patch.object(node_mod.Node, '_from_object')
|
||||
@mock.patch.object(tu, 'is_older_than')
|
||||
def test_run_health_check_healthy_internal_error(
|
||||
self, mock_tu, mock_node_obj):
|
||||
x_entity = mock.Mock()
|
||||
x_entity.do_healthcheck.side_effect = exc.InternalError(
|
||||
message='error')
|
||||
mock_node_obj.return_value = x_entity
|
||||
|
||||
ctx = mock.Mock()
|
||||
node = mock.Mock(id='FAKE_NODE1', status="ERROR",
|
||||
updated_at='2018-08-13 18:00:00',
|
||||
init_at='2018-08-13 17:00:00')
|
||||
|
||||
# do it
|
||||
res = self.hc.run_health_check(ctx, node)
|
||||
|
||||
self.assertTrue(res)
|
||||
mock_tu.assert_not_called()
|
||||
|
||||
@mock.patch.object(node_mod.Node, '_from_object')
|
||||
@mock.patch.object(tu, 'is_older_than')
|
||||
def test_run_health_check_unhealthy(self, mock_tu, mock_node_obj):
|
||||
x_entity = mock.Mock()
|
||||
x_entity.do_healthcheck.return_value = False
|
||||
mock_node_obj.return_value = x_entity
|
||||
|
||||
mock_tu.return_value = True
|
||||
|
||||
ctx = mock.Mock()
|
||||
node = mock.Mock(id='FAKE_NODE1', status="ERROR",
|
||||
updated_at='2018-08-13 18:00:00',
|
||||
init_at='2018-08-13 17:00:00')
|
||||
|
||||
# do it
|
||||
res = self.hc.run_health_check(ctx, node)
|
||||
|
||||
self.assertFalse(res)
|
||||
mock_tu.assert_called_once_with(node.updated_at, 1)
|
||||
|
||||
@mock.patch.object(node_mod.Node, '_from_object')
|
||||
@mock.patch.object(tu, 'is_older_than')
|
||||
def test_run_health_check_unhealthy_within_timeout(
|
||||
self, mock_tu, mock_node_obj):
|
||||
x_entity = mock.Mock()
|
||||
x_entity.do_healthcheck.return_value = False
|
||||
mock_node_obj.return_value = x_entity
|
||||
|
||||
mock_tu.return_value = False
|
||||
|
||||
ctx = mock.Mock()
|
||||
node = mock.Mock(id='FAKE_NODE1', status="ERROR",
|
||||
updated_at='2018-08-13 18:00:00',
|
||||
init_at='2018-08-13 17:00:00')
|
||||
|
||||
# do it
|
||||
res = self.hc.run_health_check(ctx, node)
|
||||
|
||||
self.assertTrue(res)
|
||||
mock_tu.assert_called_once_with(node.updated_at, 1)
|
||||
|
||||
|
||||
class TestNodePollUrlHealthCheck(base.SenlinTestCase):
|
||||
def setUp(self):
|
||||
super(TestNodePollUrlHealthCheck, self).setUp()
|
||||
|
|
|
@ -600,15 +600,16 @@ class TestNode(base.SenlinTestCase):
|
|||
node.status = consts.NS_ACTIVE
|
||||
node.physical_id = 'd94d6333-82e6-4f87-b7ab-b786776df9d1'
|
||||
mock_healthcheck.return_value = True
|
||||
res = node.do_healthcheck(self.context)
|
||||
res = node.do_healthcheck(self.context, consts.NODE_STATUS_POLLING)
|
||||
|
||||
self.assertTrue(res)
|
||||
mock_healthcheck.assert_called_once_with(self.context, node)
|
||||
mock_healthcheck.assert_called_once_with(self.context, node,
|
||||
consts.NODE_STATUS_POLLING)
|
||||
|
||||
def test_node_healthcheck_no_physical_id(self):
|
||||
node = nodem.Node('node1', PROFILE_ID, '')
|
||||
|
||||
res = node.do_healthcheck(self.context)
|
||||
res = node.do_healthcheck(self.context, consts.NODE_STATUS_POLLING)
|
||||
|
||||
self.assertFalse(res)
|
||||
|
||||
|
|
|
@ -35,7 +35,7 @@ class TestHealthPolicy(base.SenlinTestCase):
|
|||
|
||||
self.spec = {
|
||||
'type': 'senlin.policy.health',
|
||||
'version': '1.1',
|
||||
'version': '1.2',
|
||||
'properties': {
|
||||
'detection': {
|
||||
"detection_modes": [
|
||||
|
@ -82,7 +82,7 @@ class TestHealthPolicy(base.SenlinTestCase):
|
|||
|
||||
spec = {
|
||||
'type': 'senlin.policy.health',
|
||||
'version': '1.1',
|
||||
'version': '1.2',
|
||||
'properties': {
|
||||
'detection': {
|
||||
"detection_modes": [
|
||||
|
@ -105,7 +105,7 @@ class TestHealthPolicy(base.SenlinTestCase):
|
|||
|
||||
self.assertIsNone(hp.id)
|
||||
self.assertEqual('test-policy', hp.name)
|
||||
self.assertEqual('senlin.policy.health-1.1', hp.type)
|
||||
self.assertEqual('senlin.policy.health-1.2', hp.type)
|
||||
self.assertEqual(detection_modes, hp.detection_modes)
|
||||
self.assertEqual(60, hp.interval)
|
||||
self.assertEqual([{'name': 'REBUILD', 'params': None}],
|
||||
|
@ -114,13 +114,16 @@ class TestHealthPolicy(base.SenlinTestCase):
|
|||
def test_policy_init_ops(self):
|
||||
spec = {
|
||||
'type': 'senlin.policy.health',
|
||||
'version': '1.1',
|
||||
'version': '1.2',
|
||||
'properties': {
|
||||
'detection': {
|
||||
"detection_modes": [
|
||||
{
|
||||
'type': 'NODE_STATUS_POLLING'
|
||||
},
|
||||
{
|
||||
'type': 'HYPERVISOR_STATUS_POLLING'
|
||||
},
|
||||
{
|
||||
'type': 'NODE_STATUS_POLL_URL'
|
||||
},
|
||||
|
@ -148,7 +151,7 @@ class TestHealthPolicy(base.SenlinTestCase):
|
|||
# check result
|
||||
self.assertIsNone(hp.id)
|
||||
self.assertEqual('test-policy', hp.name)
|
||||
self.assertEqual('senlin.policy.health-1.1', hp.type)
|
||||
self.assertEqual('senlin.policy.health-1.2', hp.type)
|
||||
self.assertEqual(60, hp.interval)
|
||||
self.assertEqual([{'name': 'REBUILD', 'params': None}],
|
||||
hp.recover_actions)
|
||||
|
@ -216,7 +219,7 @@ class TestHealthPolicy(base.SenlinTestCase):
|
|||
'node_force_recreate': False,
|
||||
'recovery_conditional': 'ANY_FAILED'
|
||||
},
|
||||
'version': '1.1'
|
||||
'version': '1.2'
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -16,6 +16,7 @@ from unittest import mock
|
|||
from oslo_config import cfg
|
||||
from oslo_utils import encodeutils
|
||||
|
||||
from senlin.common import consts
|
||||
from senlin.common import exception as exc
|
||||
from senlin.objects import node as node_ob
|
||||
from senlin.profiles import base as profiles_base
|
||||
|
@ -1715,7 +1716,7 @@ class TestNovaServerBasic(base.SenlinTestCase):
|
|||
|
||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||
|
||||
res = profile.do_healthcheck(test_server)
|
||||
res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING)
|
||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||
self.assertTrue(res)
|
||||
|
||||
|
@ -1728,7 +1729,7 @@ class TestNovaServerBasic(base.SenlinTestCase):
|
|||
|
||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||
|
||||
res = profile.do_healthcheck(test_server)
|
||||
res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING)
|
||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||
self.assertTrue(res)
|
||||
|
||||
|
@ -1742,7 +1743,7 @@ class TestNovaServerBasic(base.SenlinTestCase):
|
|||
|
||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||
|
||||
res = profile.do_healthcheck(test_server)
|
||||
res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING)
|
||||
|
||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||
self.assertTrue(res)
|
||||
|
@ -1756,7 +1757,7 @@ class TestNovaServerBasic(base.SenlinTestCase):
|
|||
|
||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||
|
||||
res = profile.do_healthcheck(test_server)
|
||||
res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING)
|
||||
|
||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||
self.assertFalse(res)
|
||||
|
@ -1771,11 +1772,113 @@ class TestNovaServerBasic(base.SenlinTestCase):
|
|||
|
||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||
|
||||
res = profile.do_healthcheck(test_server)
|
||||
res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING)
|
||||
|
||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||
self.assertFalse(res)
|
||||
|
||||
def test_do_healthcheck_empty_hv_name(self):
|
||||
profile = server.ServerProfile('t', self.spec)
|
||||
|
||||
cc = mock.Mock()
|
||||
cc.hypervisor_find.return_value = None
|
||||
cc.server_get.return_value = mock.Mock(hypervisor_hostname='')
|
||||
profile._computeclient = cc
|
||||
|
||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||
|
||||
res = profile.do_healthcheck(test_server,
|
||||
consts.HYPERVISOR_STATUS_POLLING)
|
||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||
cc.hypervisor_find.assert_not_called()
|
||||
self.assertTrue(res)
|
||||
|
||||
def test_do_healthcheck_empty_hv_obj(self):
|
||||
profile = server.ServerProfile('t', self.spec)
|
||||
|
||||
cc = mock.Mock()
|
||||
cc.hypervisor_find.return_value = None
|
||||
cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV')
|
||||
profile._computeclient = cc
|
||||
|
||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||
|
||||
res = profile.do_healthcheck(test_server,
|
||||
consts.HYPERVISOR_STATUS_POLLING)
|
||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||
cc.hypervisor_find.assert_called_once_with('FAKE_HV')
|
||||
self.assertTrue(res)
|
||||
|
||||
def test_do_healthcheck_hv_exception(self):
|
||||
profile = server.ServerProfile('t', self.spec)
|
||||
|
||||
cc = mock.Mock()
|
||||
cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV')
|
||||
ex = exc.InternalError(code=503, message='Error')
|
||||
cc.hypervisor_find.side_effect = ex
|
||||
profile._computeclient = cc
|
||||
|
||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||
|
||||
res = profile.do_healthcheck(test_server,
|
||||
consts.HYPERVISOR_STATUS_POLLING)
|
||||
|
||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||
cc.hypervisor_find.assert_called_once_with('FAKE_HV')
|
||||
self.assertTrue(res)
|
||||
|
||||
def test_do_healthcheck_hv_not_found(self):
|
||||
profile = server.ServerProfile('t', self.spec)
|
||||
|
||||
cc = mock.Mock()
|
||||
cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV')
|
||||
ex = exc.InternalError(code=404, message='No Hypervisor found')
|
||||
cc.hypervisor_find.side_effect = ex
|
||||
profile._computeclient = cc
|
||||
|
||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||
|
||||
res = profile.do_healthcheck(test_server,
|
||||
consts.HYPERVISOR_STATUS_POLLING)
|
||||
|
||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||
cc.hypervisor_find.assert_called_once_with('FAKE_HV')
|
||||
self.assertFalse(res)
|
||||
|
||||
def test_do_healthcheck_hv_down(self):
|
||||
profile = server.ServerProfile('t', self.spec)
|
||||
|
||||
cc = mock.Mock()
|
||||
cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV')
|
||||
cc.hypervisor_find.return_value = mock.Mock(state='down')
|
||||
profile._computeclient = cc
|
||||
|
||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||
|
||||
res = profile.do_healthcheck(test_server,
|
||||
consts.HYPERVISOR_STATUS_POLLING)
|
||||
|
||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||
cc.hypervisor_find.assert_called_once_with('FAKE_HV')
|
||||
self.assertFalse(res)
|
||||
|
||||
def test_do_healthcheck_hv_disabled(self):
|
||||
profile = server.ServerProfile('t', self.spec)
|
||||
|
||||
cc = mock.Mock()
|
||||
cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV')
|
||||
cc.hypervisor_find.return_value = mock.Mock(status='disabled')
|
||||
profile._computeclient = cc
|
||||
|
||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||
|
||||
res = profile.do_healthcheck(test_server,
|
||||
consts.HYPERVISOR_STATUS_POLLING)
|
||||
|
||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||
cc.hypervisor_find.assert_called_once_with('FAKE_HV')
|
||||
self.assertFalse(res)
|
||||
|
||||
@mock.patch.object(server.ServerProfile, 'do_delete')
|
||||
@mock.patch.object(server.ServerProfile, 'do_create')
|
||||
def test_do_recover_operation_is_none(self, mock_create, mock_delete):
|
||||
|
|
|
@ -63,6 +63,7 @@ senlin.policies =
|
|||
senlin.policy.scaling-1.0 = senlin.policies.scaling_policy:ScalingPolicy
|
||||
senlin.policy.health-1.0 = senlin.policies.health_policy:HealthPolicy
|
||||
senlin.policy.health-1.1 = senlin.policies.health_policy:HealthPolicy
|
||||
senlin.policy.health-1.2 = senlin.policies.health_policy:HealthPolicy
|
||||
senlin.policy.loadbalance-1.0 = senlin.policies.lb_policy:LoadBalancingPolicy
|
||||
senlin.policy.loadbalance-1.1 = senlin.policies.lb_policy:LoadBalancingPolicy
|
||||
senlin.policy.loadbalance-1.2 = senlin.policies.lb_policy:LoadBalancingPolicy
|
||||
|
|
Loading…
Reference in New Issue