Add hypervisor status polling health check
Add new health check type to health policy called HYPERVISOR_STATUS_POLLING. When set, it considers a node to be unhealthy if the hypervisor on which the node is running has a disabled status or down state. Bump health policy version to 1.2 with the addition of HYPERVISOR_STATUS_POLLING. Change-Id: Ie558fb6ef7bf06cf4065fec5498e229d3146b0b2
This commit is contained in:
parent
aca5b42d81
commit
75c8450b1b
|
@ -284,9 +284,11 @@ EVENT_LEVELS = {
|
||||||
|
|
||||||
DETECTION_TYPES = (
|
DETECTION_TYPES = (
|
||||||
LIFECYCLE_EVENTS, NODE_STATUS_POLLING, NODE_STATUS_POLL_URL,
|
LIFECYCLE_EVENTS, NODE_STATUS_POLLING, NODE_STATUS_POLL_URL,
|
||||||
|
HYPERVISOR_STATUS_POLLING,
|
||||||
# LB_STATUS_POLLING,
|
# LB_STATUS_POLLING,
|
||||||
) = (
|
) = (
|
||||||
'LIFECYCLE_EVENTS', 'NODE_STATUS_POLLING', 'NODE_STATUS_POLL_URL',
|
'LIFECYCLE_EVENTS', 'NODE_STATUS_POLLING', 'NODE_STATUS_POLL_URL',
|
||||||
|
'HYPERVISOR_STATUS_POLLING',
|
||||||
# 'LB_STATUS_POLLING',
|
# 'LB_STATUS_POLLING',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -265,6 +265,41 @@ class NovaClient(base.DriverBase):
|
||||||
def hypervisor_get(self, hypervisor):
|
def hypervisor_get(self, hypervisor):
|
||||||
return self.conn.compute.get_hypervisor(hypervisor)
|
return self.conn.compute.get_hypervisor(hypervisor)
|
||||||
|
|
||||||
|
@sdk.translate_exception
|
||||||
|
def hypervisor_find(self, name_or_id, ignore_missing=False):
|
||||||
|
# try finding hypervisor by id
|
||||||
|
try:
|
||||||
|
return self.conn.compute.get_hypervisor(name_or_id)
|
||||||
|
except sdk_exc.HttpException:
|
||||||
|
# ignore http exception and instead get list and check by name
|
||||||
|
pass
|
||||||
|
|
||||||
|
# if the hypervisor could not be found using id, search list using name
|
||||||
|
results = self.conn.compute.hypervisors(
|
||||||
|
hypervisor_hostname_pattern=name_or_id)
|
||||||
|
|
||||||
|
result = None
|
||||||
|
for maybe_result in results:
|
||||||
|
name_value = maybe_result.name
|
||||||
|
|
||||||
|
if name_value == name_or_id:
|
||||||
|
# Only allow one resource to be found. If we already
|
||||||
|
# found a match, raise an exception to show it.
|
||||||
|
if result is None:
|
||||||
|
result = maybe_result
|
||||||
|
else:
|
||||||
|
msg = "More than one hypervisor exists with the name '%s'."
|
||||||
|
msg = (msg % name_or_id)
|
||||||
|
raise sdk_exc.DuplicateResource(msg)
|
||||||
|
|
||||||
|
if result is not None:
|
||||||
|
return result
|
||||||
|
|
||||||
|
if ignore_missing:
|
||||||
|
return None
|
||||||
|
raise sdk_exc.ResourceNotFound(
|
||||||
|
"No hypervisor found for %s" % (name_or_id))
|
||||||
|
|
||||||
@sdk.translate_exception
|
@sdk.translate_exception
|
||||||
def service_list(self):
|
def service_list(self):
|
||||||
return self.conn.compute.services()
|
return self.conn.compute.services()
|
||||||
|
|
|
@ -105,6 +105,9 @@ class HealthCheckType(object):
|
||||||
elif detection_type == consts.NODE_STATUS_POLL_URL:
|
elif detection_type == consts.NODE_STATUS_POLL_URL:
|
||||||
return NodePollUrlHealthCheck(
|
return NodePollUrlHealthCheck(
|
||||||
cid, interval, node_update_timeout, detection_params[0])
|
cid, interval, node_update_timeout, detection_params[0])
|
||||||
|
elif detection_type == consts.HYPERVISOR_STATUS_POLLING:
|
||||||
|
return HypervisorPollStatusHealthCheck(
|
||||||
|
cid, interval, node_update_timeout, detection_params[0])
|
||||||
else:
|
else:
|
||||||
raise Exception(
|
raise Exception(
|
||||||
'Invalid detection type: {}'.format(detection_type))
|
'Invalid detection type: {}'.format(detection_type))
|
||||||
|
@ -171,7 +174,36 @@ class NodePollStatusHealthCheck(HealthCheckType):
|
||||||
# Return False to mark the node as unhealthy if we are outside the
|
# Return False to mark the node as unhealthy if we are outside the
|
||||||
# grace period.
|
# grace period.
|
||||||
|
|
||||||
return (entity.do_healthcheck(ctx) or
|
return (entity.do_healthcheck(ctx, consts.NODE_STATUS_POLLING) or
|
||||||
|
self._node_within_grace_period(node))
|
||||||
|
except Exception as ex:
|
||||||
|
LOG.warning(
|
||||||
|
'Error when performing health check on node %s: %s',
|
||||||
|
node.id, ex
|
||||||
|
)
|
||||||
|
|
||||||
|
# treat node as healthy when an exception is encountered
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class HypervisorPollStatusHealthCheck(HealthCheckType):
|
||||||
|
def run_health_check(self, ctx, node):
|
||||||
|
"""Routine to be executed for polling hypervisor status.
|
||||||
|
|
||||||
|
:returns: True if node is healthy. False otherwise.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# create engine node from db node
|
||||||
|
entity = node_mod.Node._from_object(ctx, node)
|
||||||
|
|
||||||
|
# If health check returns True, return True to mark node as
|
||||||
|
# healthy. Else return True to mark node as healthy if we are still
|
||||||
|
# within the node's grace period to allow the node to warm-up.
|
||||||
|
# Return False to mark the node as unhealthy if we are outside the
|
||||||
|
# grace period.
|
||||||
|
|
||||||
|
return (entity.do_healthcheck(ctx,
|
||||||
|
consts.HYPERVISOR_STATUS_POLLING) or
|
||||||
self._node_within_grace_period(node))
|
self._node_within_grace_period(node))
|
||||||
except Exception as ex:
|
except Exception as ex:
|
||||||
LOG.warning(
|
LOG.warning(
|
||||||
|
@ -313,7 +345,8 @@ class HealthCheck(object):
|
||||||
|
|
||||||
def get_health_check_types(self):
|
def get_health_check_types(self):
|
||||||
polling_types = [consts.NODE_STATUS_POLLING,
|
polling_types = [consts.NODE_STATUS_POLLING,
|
||||||
consts.NODE_STATUS_POLL_URL]
|
consts.NODE_STATUS_POLL_URL,
|
||||||
|
consts.HYPERVISOR_STATUS_POLLING]
|
||||||
|
|
||||||
detection_types = self.check_type.split(',')
|
detection_types = self.check_type.split(',')
|
||||||
if all(check in polling_types for check in detection_types):
|
if all(check in polling_types for check in detection_types):
|
||||||
|
|
|
@ -349,7 +349,7 @@ class Node(object):
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def do_healthcheck(self, context):
|
def do_healthcheck(self, context, health_check_type):
|
||||||
"""health check a node.
|
"""health check a node.
|
||||||
|
|
||||||
This function is supposed to be invoked from the health manager to
|
This function is supposed to be invoked from the health manager to
|
||||||
|
@ -358,7 +358,7 @@ class Node(object):
|
||||||
:returns: True if node is healthy. False otherwise.
|
:returns: True if node is healthy. False otherwise.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
return pb.Profile.healthcheck_object(context, self)
|
return pb.Profile.healthcheck_object(context, self, health_check_type)
|
||||||
|
|
||||||
def do_recover(self, context, action):
|
def do_recover(self, context, action):
|
||||||
"""recover a node.
|
"""recover a node.
|
||||||
|
|
|
@ -29,7 +29,7 @@ LOG = logging.getLogger(__name__)
|
||||||
class HealthPolicy(base.Policy):
|
class HealthPolicy(base.Policy):
|
||||||
"""Policy for health management of a cluster."""
|
"""Policy for health management of a cluster."""
|
||||||
|
|
||||||
VERSION = '1.1'
|
VERSION = '1.2'
|
||||||
VERSIONS = {
|
VERSIONS = {
|
||||||
'1.0': [
|
'1.0': [
|
||||||
{'status': consts.EXPERIMENTAL, 'since': '2017.02'},
|
{'status': consts.EXPERIMENTAL, 'since': '2017.02'},
|
||||||
|
@ -38,6 +38,9 @@ class HealthPolicy(base.Policy):
|
||||||
'1.1': [
|
'1.1': [
|
||||||
{'status': consts.SUPPORTED, 'since': '2018.09'}
|
{'status': consts.SUPPORTED, 'since': '2018.09'}
|
||||||
],
|
],
|
||||||
|
'1.2': [
|
||||||
|
{'status': consts.SUPPORTED, 'since': '2020.09'}
|
||||||
|
],
|
||||||
}
|
}
|
||||||
PRIORITY = 600
|
PRIORITY = 600
|
||||||
|
|
||||||
|
@ -107,7 +110,7 @@ class HealthPolicy(base.Policy):
|
||||||
DETECTION_INTERVAL: schema.Integer(
|
DETECTION_INTERVAL: schema.Integer(
|
||||||
_("Number of seconds between pollings. Only "
|
_("Number of seconds between pollings. Only "
|
||||||
"required when type is 'NODE_STATUS_POLLING' or "
|
"required when type is 'NODE_STATUS_POLLING' or "
|
||||||
"'NODE_STATUS_POLL_URL'."),
|
"'NODE_STATUS_POLL_URL' or 'HYPERVISOR_STATUS_POLLING."),
|
||||||
default=60,
|
default=60,
|
||||||
),
|
),
|
||||||
NODE_UPDATE_TIMEOUT: schema.Integer(
|
NODE_UPDATE_TIMEOUT: schema.Integer(
|
||||||
|
@ -316,7 +319,8 @@ class HealthPolicy(base.Policy):
|
||||||
|
|
||||||
# check valid detection types
|
# check valid detection types
|
||||||
polling_types = [consts.NODE_STATUS_POLLING,
|
polling_types = [consts.NODE_STATUS_POLLING,
|
||||||
consts.NODE_STATUS_POLL_URL]
|
consts.NODE_STATUS_POLL_URL,
|
||||||
|
consts.HYPERVISOR_STATUS_POLLING]
|
||||||
|
|
||||||
has_valid_polling_types = all(
|
has_valid_polling_types = all(
|
||||||
d.type in polling_types
|
d.type in polling_types
|
||||||
|
|
|
@ -305,9 +305,9 @@ class Profile(object):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@profiler.trace('Profile.check_object', hide_args=False)
|
@profiler.trace('Profile.check_object', hide_args=False)
|
||||||
def healthcheck_object(cls, ctx, obj):
|
def healthcheck_object(cls, ctx, obj, health_check_type):
|
||||||
profile = cls.load(ctx, profile_id=obj.profile_id)
|
profile = cls.load(ctx, profile_id=obj.profile_id)
|
||||||
return profile.do_healthcheck(obj)
|
return profile.do_healthcheck(obj, health_check_type)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@profiler.trace('Profile.recover_object', hide_args=False)
|
@profiler.trace('Profile.recover_object', hide_args=False)
|
||||||
|
|
|
@ -1638,22 +1638,20 @@ class ServerProfile(base.Profile):
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def do_healthcheck(self, obj):
|
def do_healthcheck(self, obj, health_check_type):
|
||||||
"""Healthcheck operation.
|
"""Healthcheck operation.
|
||||||
|
|
||||||
This method checks if a server node is healthy by getting the server
|
This method checks if a node is healthy. If health check type is
|
||||||
status from nova. A server is considered unhealthy if it does not
|
NODE_STATUS_POLLING it will check the server status. If health check
|
||||||
exist or its status is one of the following:
|
type is HYPERVISOR_STATUS_POLLING it will check the hypervisor state
|
||||||
- ERROR
|
and status.
|
||||||
- SHUTOFF
|
|
||||||
- DELETED
|
|
||||||
|
|
||||||
:param obj: The node object to operate on.
|
:param obj: The node object to operate on.
|
||||||
|
:param health_check_type: The type of health check. Either
|
||||||
|
NODE_STATUS_POLLING or HYPERVISOR_STATUS_POLLING.
|
||||||
:return status: True indicates node is healthy, False indicates
|
:return status: True indicates node is healthy, False indicates
|
||||||
it is unhealthy.
|
it is unhealthy.
|
||||||
"""
|
"""
|
||||||
unhealthy_server_status = [consts.VS_ERROR, consts.VS_SHUTOFF,
|
|
||||||
consts.VS_DELETED]
|
|
||||||
|
|
||||||
if not obj.physical_id:
|
if not obj.physical_id:
|
||||||
if obj.status == 'BUILD' or obj.status == 'CREATING':
|
if obj.status == 'BUILD' or obj.status == 'CREATING':
|
||||||
|
@ -1687,6 +1685,34 @@ class ServerProfile(base.Profile):
|
||||||
consts.POLL_STATUS_PASS, obj.name)
|
consts.POLL_STATUS_PASS, obj.name)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
if health_check_type == consts.NODE_STATUS_POLLING:
|
||||||
|
return self._do_healthcheck_server(obj, server)
|
||||||
|
elif health_check_type == consts.HYPERVISOR_STATUS_POLLING:
|
||||||
|
return self._do_healthcheck_hypervisor(obj, server)
|
||||||
|
else:
|
||||||
|
LOG.info('%s for %s: ignoring invalid health check type %s',
|
||||||
|
consts.POLL_STATUS_PASS, obj.name, health_check_type)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _do_healthcheck_server(self, obj, server):
|
||||||
|
"""Healthcheck operation based on server.
|
||||||
|
|
||||||
|
This method checks if a server node is healthy by getting the server
|
||||||
|
status from nova. A server is considered unhealthy if it does not
|
||||||
|
exist or its status is one of the following:
|
||||||
|
- ERROR
|
||||||
|
- SHUTOFF
|
||||||
|
- DELETED
|
||||||
|
|
||||||
|
:param obj: The node object to operate on.
|
||||||
|
:param server: The server object associated with the node.
|
||||||
|
:return status: True indicates node is healthy, False indicates
|
||||||
|
it is unhealthy.
|
||||||
|
"""
|
||||||
|
|
||||||
|
unhealthy_server_status = [consts.VS_ERROR, consts.VS_SHUTOFF,
|
||||||
|
consts.VS_DELETED]
|
||||||
|
|
||||||
if server.status in unhealthy_server_status:
|
if server.status in unhealthy_server_status:
|
||||||
LOG.info('%s for %s: server status is unhealthy.',
|
LOG.info('%s for %s: server status is unhealthy.',
|
||||||
consts.POLL_STATUS_FAIL, obj.name)
|
consts.POLL_STATUS_FAIL, obj.name)
|
||||||
|
@ -1695,6 +1721,66 @@ class ServerProfile(base.Profile):
|
||||||
LOG.info('%s for %s', consts.POLL_STATUS_PASS, obj.name)
|
LOG.info('%s for %s', consts.POLL_STATUS_PASS, obj.name)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def _do_healthcheck_hypervisor(self, obj, server):
|
||||||
|
"""Healthcheck operation based on hypervisor.
|
||||||
|
|
||||||
|
This method checks if a server node is healthy by getting the
|
||||||
|
hypervisor state and status from nova. A server is considered
|
||||||
|
unhealthy if the hypervisor it is running on has a state that is down
|
||||||
|
or a status that is disabled.
|
||||||
|
|
||||||
|
:param obj: The node object to operate on.
|
||||||
|
:param server: The server object associated with the node.
|
||||||
|
:return status: True indicates node is healthy, False indicates
|
||||||
|
it is unhealthy.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if server.hypervisor_hostname != "":
|
||||||
|
try:
|
||||||
|
hv = self.compute(obj).hypervisor_find(
|
||||||
|
server.hypervisor_hostname)
|
||||||
|
except Exception as ex:
|
||||||
|
if isinstance(ex, exc.InternalError) and ex.code == 404:
|
||||||
|
# treat resource not found exception as unhealthy
|
||||||
|
LOG.info('%s for %s: hypervisor %s was not found.',
|
||||||
|
consts.POLL_STATUS_FAIL, obj.name,
|
||||||
|
server.hypervisor_hostname)
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
# treat all other exceptions as healthy
|
||||||
|
LOG.info(
|
||||||
|
'%s for %s: Exception when trying to get hypervisor '
|
||||||
|
'info for %s, but ignoring this error: %s.',
|
||||||
|
consts.POLL_STATUS_PASS, obj.name,
|
||||||
|
server.hypervisor_hostname, ex.message)
|
||||||
|
return True
|
||||||
|
|
||||||
|
if hv is None:
|
||||||
|
# no hypervisor information is available, treat the node as
|
||||||
|
# healthy
|
||||||
|
LOG.info(
|
||||||
|
'%s for %s: No hypervisor information was returned but '
|
||||||
|
'ignoring this error.',
|
||||||
|
consts.POLL_STATUS_PASS, obj.name)
|
||||||
|
return True
|
||||||
|
|
||||||
|
if hv.state == 'down':
|
||||||
|
LOG.info('%s for %s: server status is unhealthy because '
|
||||||
|
'hypervisor %s state is down',
|
||||||
|
consts.POLL_STATUS_FAIL, obj.name,
|
||||||
|
server.hypervisor_hostname)
|
||||||
|
return False
|
||||||
|
|
||||||
|
if hv.status == 'disabled':
|
||||||
|
LOG.info('%s for %s: server status is unhealthy because '
|
||||||
|
'hypervisor %s status is disabled',
|
||||||
|
consts.POLL_STATUS_FAIL, obj.name,
|
||||||
|
server.hypervisor_hostname)
|
||||||
|
return False
|
||||||
|
|
||||||
|
LOG.info('%s for %s', consts.POLL_STATUS_PASS, obj.name)
|
||||||
|
return True
|
||||||
|
|
||||||
def do_recover(self, obj, **options):
|
def do_recover(self, obj, **options):
|
||||||
"""Handler for recover operation.
|
"""Handler for recover operation.
|
||||||
|
|
||||||
|
|
|
@ -15,6 +15,7 @@ from unittest import mock
|
||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
from senlin.common import exception as exc
|
||||||
from senlin.drivers.os import nova_v2
|
from senlin.drivers.os import nova_v2
|
||||||
from senlin.drivers import sdk
|
from senlin.drivers import sdk
|
||||||
from senlin.tests.unit.common import base
|
from senlin.tests.unit.common import base
|
||||||
|
@ -552,6 +553,65 @@ class TestNovaV2(base.SenlinTestCase):
|
||||||
d.hypervisor_get('k')
|
d.hypervisor_get('k')
|
||||||
self.compute.get_hypervisor.assert_called_once_with('k')
|
self.compute.get_hypervisor.assert_called_once_with('k')
|
||||||
|
|
||||||
|
def test_hypervisor_find(self):
|
||||||
|
d = nova_v2.NovaClient(self.conn_params)
|
||||||
|
self.compute.get_hypervisor.return_value = mock.Mock()
|
||||||
|
|
||||||
|
d.hypervisor_find('k')
|
||||||
|
|
||||||
|
self.compute.get_hypervisor.assert_called_once_with('k')
|
||||||
|
|
||||||
|
def test_hypervisor_find_name(self):
|
||||||
|
d = nova_v2.NovaClient(self.conn_params)
|
||||||
|
self.compute.get_hypervisor.side_effect = sdk_exc.HttpException
|
||||||
|
fake_result = mock.Mock()
|
||||||
|
fake_result.name = 'FAKE_HV'
|
||||||
|
self.compute.hypervisors.return_value = [mock.Mock(name='not_it'),
|
||||||
|
fake_result]
|
||||||
|
|
||||||
|
r = d.hypervisor_find('FAKE_HV')
|
||||||
|
|
||||||
|
self.compute.get_hypervisor.assert_called_once_with('FAKE_HV')
|
||||||
|
self.compute.hypervisors.assert_called_once_with(
|
||||||
|
hypervisor_hostname_pattern='FAKE_HV')
|
||||||
|
self.assertEqual(r, fake_result)
|
||||||
|
|
||||||
|
def test_hypervisor_find_name_duplicate(self):
|
||||||
|
d = nova_v2.NovaClient(self.conn_params)
|
||||||
|
self.compute.get_hypervisor.side_effect = sdk_exc.HttpException
|
||||||
|
fake_result = mock.Mock()
|
||||||
|
fake_result.name = 'FAKE_HV'
|
||||||
|
self.compute.hypervisors.return_value = [fake_result, fake_result]
|
||||||
|
|
||||||
|
self.assertRaises(exc.InternalError, d.hypervisor_find, 'FAKE_HV')
|
||||||
|
|
||||||
|
self.compute.get_hypervisor.assert_called_once_with('FAKE_HV')
|
||||||
|
self.compute.hypervisors.assert_called_once_with(
|
||||||
|
hypervisor_hostname_pattern='FAKE_HV')
|
||||||
|
|
||||||
|
def test_hypervisor_find_name_ignore_missing(self):
|
||||||
|
d = nova_v2.NovaClient(self.conn_params)
|
||||||
|
self.compute.get_hypervisor.side_effect = sdk_exc.HttpException
|
||||||
|
self.compute.hypervisors.return_value = [mock.Mock(name='not_it')]
|
||||||
|
|
||||||
|
r = d.hypervisor_find('FAKE_HV', True)
|
||||||
|
|
||||||
|
self.compute.get_hypervisor.assert_called_once_with('FAKE_HV')
|
||||||
|
self.compute.hypervisors.assert_called_once_with(
|
||||||
|
hypervisor_hostname_pattern='FAKE_HV')
|
||||||
|
self.assertIsNone(r)
|
||||||
|
|
||||||
|
def test_hypervisor_find_name_not_found(self):
|
||||||
|
d = nova_v2.NovaClient(self.conn_params)
|
||||||
|
self.compute.get_hypervisor.side_effect = sdk_exc.HttpException
|
||||||
|
self.compute.hypervisors.return_value = [mock.Mock(name='not_it')]
|
||||||
|
|
||||||
|
self.assertRaises(exc.InternalError, d.hypervisor_find, 'FAKE_HV')
|
||||||
|
|
||||||
|
self.compute.get_hypervisor.assert_called_once_with('FAKE_HV')
|
||||||
|
self.compute.hypervisors.assert_called_once_with(
|
||||||
|
hypervisor_hostname_pattern='FAKE_HV')
|
||||||
|
|
||||||
def test_service_list(self):
|
def test_service_list(self):
|
||||||
d = nova_v2.NovaClient(self.conn_params)
|
d = nova_v2.NovaClient(self.conn_params)
|
||||||
d.service_list()
|
d.service_list()
|
||||||
|
|
|
@ -326,6 +326,15 @@ class TestHealthCheckType(base.SenlinTestCase):
|
||||||
'poll_url_retry_limit': '',
|
'poll_url_retry_limit': '',
|
||||||
'poll_url_retry_interval': ''
|
'poll_url_retry_interval': ''
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
'type': 'HYPERVISOR_STATUS_POLLING',
|
||||||
|
'poll_url': '',
|
||||||
|
'poll_url_ssl_verify': True,
|
||||||
|
'poll_url_conn_error_as_unhealthy': True,
|
||||||
|
'poll_url_healthy_response': '',
|
||||||
|
'poll_url_retry_limit': '',
|
||||||
|
'poll_url_retry_interval': ''
|
||||||
|
},
|
||||||
{
|
{
|
||||||
'type': 'NODE_STATUS_POLL_URL',
|
'type': 'NODE_STATUS_POLL_URL',
|
||||||
'poll_url': '',
|
'poll_url': '',
|
||||||
|
@ -492,6 +501,95 @@ class TestNodePollStatusHealthCheck(base.SenlinTestCase):
|
||||||
mock_tu.assert_called_once_with(node.updated_at, 1)
|
mock_tu.assert_called_once_with(node.updated_at, 1)
|
||||||
|
|
||||||
|
|
||||||
|
class TestHypervisorPollStatusHealthCheck(base.SenlinTestCase):
|
||||||
|
def setUp(self):
|
||||||
|
super(TestHypervisorPollStatusHealthCheck, self).setUp()
|
||||||
|
|
||||||
|
self.hc = hm.HypervisorPollStatusHealthCheck(
|
||||||
|
cluster_id='CLUSTER_ID',
|
||||||
|
interval=1, node_update_timeout=1, params=''
|
||||||
|
)
|
||||||
|
|
||||||
|
@mock.patch.object(node_mod.Node, '_from_object')
|
||||||
|
@mock.patch.object(tu, 'is_older_than')
|
||||||
|
def test_run_health_check_healthy(self, mock_tu, mock_node_obj):
|
||||||
|
x_entity = mock.Mock()
|
||||||
|
x_entity.do_healthcheck.return_value = True
|
||||||
|
mock_node_obj.return_value = x_entity
|
||||||
|
|
||||||
|
ctx = mock.Mock()
|
||||||
|
node = mock.Mock(id='FAKE_NODE1', status="ERROR",
|
||||||
|
updated_at='2018-08-13 18:00:00',
|
||||||
|
init_at='2018-08-13 17:00:00')
|
||||||
|
|
||||||
|
# do it
|
||||||
|
res = self.hc.run_health_check(ctx, node)
|
||||||
|
|
||||||
|
self.assertTrue(res)
|
||||||
|
mock_tu.assert_not_called()
|
||||||
|
|
||||||
|
@mock.patch.object(node_mod.Node, '_from_object')
|
||||||
|
@mock.patch.object(tu, 'is_older_than')
|
||||||
|
def test_run_health_check_healthy_internal_error(
|
||||||
|
self, mock_tu, mock_node_obj):
|
||||||
|
x_entity = mock.Mock()
|
||||||
|
x_entity.do_healthcheck.side_effect = exc.InternalError(
|
||||||
|
message='error')
|
||||||
|
mock_node_obj.return_value = x_entity
|
||||||
|
|
||||||
|
ctx = mock.Mock()
|
||||||
|
node = mock.Mock(id='FAKE_NODE1', status="ERROR",
|
||||||
|
updated_at='2018-08-13 18:00:00',
|
||||||
|
init_at='2018-08-13 17:00:00')
|
||||||
|
|
||||||
|
# do it
|
||||||
|
res = self.hc.run_health_check(ctx, node)
|
||||||
|
|
||||||
|
self.assertTrue(res)
|
||||||
|
mock_tu.assert_not_called()
|
||||||
|
|
||||||
|
@mock.patch.object(node_mod.Node, '_from_object')
|
||||||
|
@mock.patch.object(tu, 'is_older_than')
|
||||||
|
def test_run_health_check_unhealthy(self, mock_tu, mock_node_obj):
|
||||||
|
x_entity = mock.Mock()
|
||||||
|
x_entity.do_healthcheck.return_value = False
|
||||||
|
mock_node_obj.return_value = x_entity
|
||||||
|
|
||||||
|
mock_tu.return_value = True
|
||||||
|
|
||||||
|
ctx = mock.Mock()
|
||||||
|
node = mock.Mock(id='FAKE_NODE1', status="ERROR",
|
||||||
|
updated_at='2018-08-13 18:00:00',
|
||||||
|
init_at='2018-08-13 17:00:00')
|
||||||
|
|
||||||
|
# do it
|
||||||
|
res = self.hc.run_health_check(ctx, node)
|
||||||
|
|
||||||
|
self.assertFalse(res)
|
||||||
|
mock_tu.assert_called_once_with(node.updated_at, 1)
|
||||||
|
|
||||||
|
@mock.patch.object(node_mod.Node, '_from_object')
|
||||||
|
@mock.patch.object(tu, 'is_older_than')
|
||||||
|
def test_run_health_check_unhealthy_within_timeout(
|
||||||
|
self, mock_tu, mock_node_obj):
|
||||||
|
x_entity = mock.Mock()
|
||||||
|
x_entity.do_healthcheck.return_value = False
|
||||||
|
mock_node_obj.return_value = x_entity
|
||||||
|
|
||||||
|
mock_tu.return_value = False
|
||||||
|
|
||||||
|
ctx = mock.Mock()
|
||||||
|
node = mock.Mock(id='FAKE_NODE1', status="ERROR",
|
||||||
|
updated_at='2018-08-13 18:00:00',
|
||||||
|
init_at='2018-08-13 17:00:00')
|
||||||
|
|
||||||
|
# do it
|
||||||
|
res = self.hc.run_health_check(ctx, node)
|
||||||
|
|
||||||
|
self.assertTrue(res)
|
||||||
|
mock_tu.assert_called_once_with(node.updated_at, 1)
|
||||||
|
|
||||||
|
|
||||||
class TestNodePollUrlHealthCheck(base.SenlinTestCase):
|
class TestNodePollUrlHealthCheck(base.SenlinTestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(TestNodePollUrlHealthCheck, self).setUp()
|
super(TestNodePollUrlHealthCheck, self).setUp()
|
||||||
|
|
|
@ -600,15 +600,16 @@ class TestNode(base.SenlinTestCase):
|
||||||
node.status = consts.NS_ACTIVE
|
node.status = consts.NS_ACTIVE
|
||||||
node.physical_id = 'd94d6333-82e6-4f87-b7ab-b786776df9d1'
|
node.physical_id = 'd94d6333-82e6-4f87-b7ab-b786776df9d1'
|
||||||
mock_healthcheck.return_value = True
|
mock_healthcheck.return_value = True
|
||||||
res = node.do_healthcheck(self.context)
|
res = node.do_healthcheck(self.context, consts.NODE_STATUS_POLLING)
|
||||||
|
|
||||||
self.assertTrue(res)
|
self.assertTrue(res)
|
||||||
mock_healthcheck.assert_called_once_with(self.context, node)
|
mock_healthcheck.assert_called_once_with(self.context, node,
|
||||||
|
consts.NODE_STATUS_POLLING)
|
||||||
|
|
||||||
def test_node_healthcheck_no_physical_id(self):
|
def test_node_healthcheck_no_physical_id(self):
|
||||||
node = nodem.Node('node1', PROFILE_ID, '')
|
node = nodem.Node('node1', PROFILE_ID, '')
|
||||||
|
|
||||||
res = node.do_healthcheck(self.context)
|
res = node.do_healthcheck(self.context, consts.NODE_STATUS_POLLING)
|
||||||
|
|
||||||
self.assertFalse(res)
|
self.assertFalse(res)
|
||||||
|
|
||||||
|
|
|
@ -35,7 +35,7 @@ class TestHealthPolicy(base.SenlinTestCase):
|
||||||
|
|
||||||
self.spec = {
|
self.spec = {
|
||||||
'type': 'senlin.policy.health',
|
'type': 'senlin.policy.health',
|
||||||
'version': '1.1',
|
'version': '1.2',
|
||||||
'properties': {
|
'properties': {
|
||||||
'detection': {
|
'detection': {
|
||||||
"detection_modes": [
|
"detection_modes": [
|
||||||
|
@ -82,7 +82,7 @@ class TestHealthPolicy(base.SenlinTestCase):
|
||||||
|
|
||||||
spec = {
|
spec = {
|
||||||
'type': 'senlin.policy.health',
|
'type': 'senlin.policy.health',
|
||||||
'version': '1.1',
|
'version': '1.2',
|
||||||
'properties': {
|
'properties': {
|
||||||
'detection': {
|
'detection': {
|
||||||
"detection_modes": [
|
"detection_modes": [
|
||||||
|
@ -105,7 +105,7 @@ class TestHealthPolicy(base.SenlinTestCase):
|
||||||
|
|
||||||
self.assertIsNone(hp.id)
|
self.assertIsNone(hp.id)
|
||||||
self.assertEqual('test-policy', hp.name)
|
self.assertEqual('test-policy', hp.name)
|
||||||
self.assertEqual('senlin.policy.health-1.1', hp.type)
|
self.assertEqual('senlin.policy.health-1.2', hp.type)
|
||||||
self.assertEqual(detection_modes, hp.detection_modes)
|
self.assertEqual(detection_modes, hp.detection_modes)
|
||||||
self.assertEqual(60, hp.interval)
|
self.assertEqual(60, hp.interval)
|
||||||
self.assertEqual([{'name': 'REBUILD', 'params': None}],
|
self.assertEqual([{'name': 'REBUILD', 'params': None}],
|
||||||
|
@ -114,13 +114,16 @@ class TestHealthPolicy(base.SenlinTestCase):
|
||||||
def test_policy_init_ops(self):
|
def test_policy_init_ops(self):
|
||||||
spec = {
|
spec = {
|
||||||
'type': 'senlin.policy.health',
|
'type': 'senlin.policy.health',
|
||||||
'version': '1.1',
|
'version': '1.2',
|
||||||
'properties': {
|
'properties': {
|
||||||
'detection': {
|
'detection': {
|
||||||
"detection_modes": [
|
"detection_modes": [
|
||||||
{
|
{
|
||||||
'type': 'NODE_STATUS_POLLING'
|
'type': 'NODE_STATUS_POLLING'
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
'type': 'HYPERVISOR_STATUS_POLLING'
|
||||||
|
},
|
||||||
{
|
{
|
||||||
'type': 'NODE_STATUS_POLL_URL'
|
'type': 'NODE_STATUS_POLL_URL'
|
||||||
},
|
},
|
||||||
|
@ -148,7 +151,7 @@ class TestHealthPolicy(base.SenlinTestCase):
|
||||||
# check result
|
# check result
|
||||||
self.assertIsNone(hp.id)
|
self.assertIsNone(hp.id)
|
||||||
self.assertEqual('test-policy', hp.name)
|
self.assertEqual('test-policy', hp.name)
|
||||||
self.assertEqual('senlin.policy.health-1.1', hp.type)
|
self.assertEqual('senlin.policy.health-1.2', hp.type)
|
||||||
self.assertEqual(60, hp.interval)
|
self.assertEqual(60, hp.interval)
|
||||||
self.assertEqual([{'name': 'REBUILD', 'params': None}],
|
self.assertEqual([{'name': 'REBUILD', 'params': None}],
|
||||||
hp.recover_actions)
|
hp.recover_actions)
|
||||||
|
@ -216,7 +219,7 @@ class TestHealthPolicy(base.SenlinTestCase):
|
||||||
'node_force_recreate': False,
|
'node_force_recreate': False,
|
||||||
'recovery_conditional': 'ANY_FAILED'
|
'recovery_conditional': 'ANY_FAILED'
|
||||||
},
|
},
|
||||||
'version': '1.1'
|
'version': '1.2'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -16,6 +16,7 @@ from unittest import mock
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_utils import encodeutils
|
from oslo_utils import encodeutils
|
||||||
|
|
||||||
|
from senlin.common import consts
|
||||||
from senlin.common import exception as exc
|
from senlin.common import exception as exc
|
||||||
from senlin.objects import node as node_ob
|
from senlin.objects import node as node_ob
|
||||||
from senlin.profiles import base as profiles_base
|
from senlin.profiles import base as profiles_base
|
||||||
|
@ -1715,7 +1716,7 @@ class TestNovaServerBasic(base.SenlinTestCase):
|
||||||
|
|
||||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||||
|
|
||||||
res = profile.do_healthcheck(test_server)
|
res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING)
|
||||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||||
self.assertTrue(res)
|
self.assertTrue(res)
|
||||||
|
|
||||||
|
@ -1728,7 +1729,7 @@ class TestNovaServerBasic(base.SenlinTestCase):
|
||||||
|
|
||||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||||
|
|
||||||
res = profile.do_healthcheck(test_server)
|
res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING)
|
||||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||||
self.assertTrue(res)
|
self.assertTrue(res)
|
||||||
|
|
||||||
|
@ -1742,7 +1743,7 @@ class TestNovaServerBasic(base.SenlinTestCase):
|
||||||
|
|
||||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||||
|
|
||||||
res = profile.do_healthcheck(test_server)
|
res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING)
|
||||||
|
|
||||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||||
self.assertTrue(res)
|
self.assertTrue(res)
|
||||||
|
@ -1756,7 +1757,7 @@ class TestNovaServerBasic(base.SenlinTestCase):
|
||||||
|
|
||||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||||
|
|
||||||
res = profile.do_healthcheck(test_server)
|
res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING)
|
||||||
|
|
||||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||||
self.assertFalse(res)
|
self.assertFalse(res)
|
||||||
|
@ -1771,11 +1772,113 @@ class TestNovaServerBasic(base.SenlinTestCase):
|
||||||
|
|
||||||
test_server = mock.Mock(physical_id='FAKE_ID')
|
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||||
|
|
||||||
res = profile.do_healthcheck(test_server)
|
res = profile.do_healthcheck(test_server, consts.NODE_STATUS_POLLING)
|
||||||
|
|
||||||
cc.server_get.assert_called_once_with('FAKE_ID')
|
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||||
self.assertFalse(res)
|
self.assertFalse(res)
|
||||||
|
|
||||||
|
def test_do_healthcheck_empty_hv_name(self):
|
||||||
|
profile = server.ServerProfile('t', self.spec)
|
||||||
|
|
||||||
|
cc = mock.Mock()
|
||||||
|
cc.hypervisor_find.return_value = None
|
||||||
|
cc.server_get.return_value = mock.Mock(hypervisor_hostname='')
|
||||||
|
profile._computeclient = cc
|
||||||
|
|
||||||
|
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||||
|
|
||||||
|
res = profile.do_healthcheck(test_server,
|
||||||
|
consts.HYPERVISOR_STATUS_POLLING)
|
||||||
|
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||||
|
cc.hypervisor_find.assert_not_called()
|
||||||
|
self.assertTrue(res)
|
||||||
|
|
||||||
|
def test_do_healthcheck_empty_hv_obj(self):
|
||||||
|
profile = server.ServerProfile('t', self.spec)
|
||||||
|
|
||||||
|
cc = mock.Mock()
|
||||||
|
cc.hypervisor_find.return_value = None
|
||||||
|
cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV')
|
||||||
|
profile._computeclient = cc
|
||||||
|
|
||||||
|
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||||
|
|
||||||
|
res = profile.do_healthcheck(test_server,
|
||||||
|
consts.HYPERVISOR_STATUS_POLLING)
|
||||||
|
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||||
|
cc.hypervisor_find.assert_called_once_with('FAKE_HV')
|
||||||
|
self.assertTrue(res)
|
||||||
|
|
||||||
|
def test_do_healthcheck_hv_exception(self):
|
||||||
|
profile = server.ServerProfile('t', self.spec)
|
||||||
|
|
||||||
|
cc = mock.Mock()
|
||||||
|
cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV')
|
||||||
|
ex = exc.InternalError(code=503, message='Error')
|
||||||
|
cc.hypervisor_find.side_effect = ex
|
||||||
|
profile._computeclient = cc
|
||||||
|
|
||||||
|
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||||
|
|
||||||
|
res = profile.do_healthcheck(test_server,
|
||||||
|
consts.HYPERVISOR_STATUS_POLLING)
|
||||||
|
|
||||||
|
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||||
|
cc.hypervisor_find.assert_called_once_with('FAKE_HV')
|
||||||
|
self.assertTrue(res)
|
||||||
|
|
||||||
|
def test_do_healthcheck_hv_not_found(self):
|
||||||
|
profile = server.ServerProfile('t', self.spec)
|
||||||
|
|
||||||
|
cc = mock.Mock()
|
||||||
|
cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV')
|
||||||
|
ex = exc.InternalError(code=404, message='No Hypervisor found')
|
||||||
|
cc.hypervisor_find.side_effect = ex
|
||||||
|
profile._computeclient = cc
|
||||||
|
|
||||||
|
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||||
|
|
||||||
|
res = profile.do_healthcheck(test_server,
|
||||||
|
consts.HYPERVISOR_STATUS_POLLING)
|
||||||
|
|
||||||
|
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||||
|
cc.hypervisor_find.assert_called_once_with('FAKE_HV')
|
||||||
|
self.assertFalse(res)
|
||||||
|
|
||||||
|
def test_do_healthcheck_hv_down(self):
|
||||||
|
profile = server.ServerProfile('t', self.spec)
|
||||||
|
|
||||||
|
cc = mock.Mock()
|
||||||
|
cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV')
|
||||||
|
cc.hypervisor_find.return_value = mock.Mock(state='down')
|
||||||
|
profile._computeclient = cc
|
||||||
|
|
||||||
|
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||||
|
|
||||||
|
res = profile.do_healthcheck(test_server,
|
||||||
|
consts.HYPERVISOR_STATUS_POLLING)
|
||||||
|
|
||||||
|
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||||
|
cc.hypervisor_find.assert_called_once_with('FAKE_HV')
|
||||||
|
self.assertFalse(res)
|
||||||
|
|
||||||
|
def test_do_healthcheck_hv_disabled(self):
|
||||||
|
profile = server.ServerProfile('t', self.spec)
|
||||||
|
|
||||||
|
cc = mock.Mock()
|
||||||
|
cc.server_get.return_value = mock.Mock(hypervisor_hostname='FAKE_HV')
|
||||||
|
cc.hypervisor_find.return_value = mock.Mock(status='disabled')
|
||||||
|
profile._computeclient = cc
|
||||||
|
|
||||||
|
test_server = mock.Mock(physical_id='FAKE_ID')
|
||||||
|
|
||||||
|
res = profile.do_healthcheck(test_server,
|
||||||
|
consts.HYPERVISOR_STATUS_POLLING)
|
||||||
|
|
||||||
|
cc.server_get.assert_called_once_with('FAKE_ID')
|
||||||
|
cc.hypervisor_find.assert_called_once_with('FAKE_HV')
|
||||||
|
self.assertFalse(res)
|
||||||
|
|
||||||
@mock.patch.object(server.ServerProfile, 'do_delete')
|
@mock.patch.object(server.ServerProfile, 'do_delete')
|
||||||
@mock.patch.object(server.ServerProfile, 'do_create')
|
@mock.patch.object(server.ServerProfile, 'do_create')
|
||||||
def test_do_recover_operation_is_none(self, mock_create, mock_delete):
|
def test_do_recover_operation_is_none(self, mock_create, mock_delete):
|
||||||
|
|
|
@ -62,6 +62,7 @@ senlin.policies =
|
||||||
senlin.policy.scaling-1.0 = senlin.policies.scaling_policy:ScalingPolicy
|
senlin.policy.scaling-1.0 = senlin.policies.scaling_policy:ScalingPolicy
|
||||||
senlin.policy.health-1.0 = senlin.policies.health_policy:HealthPolicy
|
senlin.policy.health-1.0 = senlin.policies.health_policy:HealthPolicy
|
||||||
senlin.policy.health-1.1 = senlin.policies.health_policy:HealthPolicy
|
senlin.policy.health-1.1 = senlin.policies.health_policy:HealthPolicy
|
||||||
|
senlin.policy.health-1.2 = senlin.policies.health_policy:HealthPolicy
|
||||||
senlin.policy.loadbalance-1.0 = senlin.policies.lb_policy:LoadBalancingPolicy
|
senlin.policy.loadbalance-1.0 = senlin.policies.lb_policy:LoadBalancingPolicy
|
||||||
senlin.policy.loadbalance-1.1 = senlin.policies.lb_policy:LoadBalancingPolicy
|
senlin.policy.loadbalance-1.1 = senlin.policies.lb_policy:LoadBalancingPolicy
|
||||||
senlin.policy.loadbalance-1.2 = senlin.policies.lb_policy:LoadBalancingPolicy
|
senlin.policy.loadbalance-1.2 = senlin.policies.lb_policy:LoadBalancingPolicy
|
||||||
|
|
Loading…
Reference in New Issue