Getting the heartbeat from Ironic instead
This commit is contained in:
parent
0e6776f6c0
commit
6dc09f3148
ironic_python_agent
@ -81,16 +81,17 @@ class IronicPythonAgentHeartbeater(threading.Thread):
|
|||||||
interval = 0
|
interval = 0
|
||||||
|
|
||||||
while not self.stop_event.wait(interval):
|
while not self.stop_event.wait(interval):
|
||||||
next_heartbeat_by = self.do_heartbeat()
|
self.do_heartbeat()
|
||||||
|
# next_heartbeat_by = _time() + self.agent.heartbeat_timeout
|
||||||
interval_multiplier = random.uniform(self.min_jitter_multiplier,
|
interval_multiplier = random.uniform(self.min_jitter_multiplier,
|
||||||
self.max_jitter_multiplier)
|
self.max_jitter_multiplier)
|
||||||
interval = (next_heartbeat_by - _time()) * interval_multiplier
|
interval = self.agent.heartbeat_timeout * interval_multiplier
|
||||||
log_msg = 'sleeping before next heartbeat, interval: {0}'
|
log_msg = 'sleeping before next heartbeat, interval: {0}'
|
||||||
self.log.info(log_msg.format(interval))
|
self.log.info(log_msg.format(interval))
|
||||||
|
|
||||||
def do_heartbeat(self):
|
def do_heartbeat(self):
|
||||||
try:
|
try:
|
||||||
deadline = self.api.heartbeat(
|
self.api.heartbeat(
|
||||||
uuid=self.agent.get_node_uuid(),
|
uuid=self.agent.get_node_uuid(),
|
||||||
advertise_address=self.agent.advertise_address
|
advertise_address=self.agent.advertise_address
|
||||||
)
|
)
|
||||||
@ -98,12 +99,8 @@ class IronicPythonAgentHeartbeater(threading.Thread):
|
|||||||
self.log.info('heartbeat successful')
|
self.log.info('heartbeat successful')
|
||||||
except Exception:
|
except Exception:
|
||||||
self.log.exception('error sending heartbeat')
|
self.log.exception('error sending heartbeat')
|
||||||
deadline = _time() + self.error_delay
|
|
||||||
self.error_delay = min(self.error_delay * self.backoff_factor,
|
self.error_delay = min(self.error_delay * self.backoff_factor,
|
||||||
self.max_delay)
|
self.max_delay)
|
||||||
pass
|
|
||||||
|
|
||||||
return deadline
|
|
||||||
|
|
||||||
def stop(self):
|
def stop(self):
|
||||||
self.log.info('stopping heartbeater')
|
self.log.info('stopping heartbeater')
|
||||||
@ -123,6 +120,7 @@ class IronicPythonAgent(object):
|
|||||||
self.api = app.VersionSelectorApplication(self)
|
self.api = app.VersionSelectorApplication(self)
|
||||||
self.command_results = utils.get_ordereddict()
|
self.command_results = utils.get_ordereddict()
|
||||||
self.heartbeater = IronicPythonAgentHeartbeater(self)
|
self.heartbeater = IronicPythonAgentHeartbeater(self)
|
||||||
|
self.heartbeat_timeout = None
|
||||||
self.hardware = hardware.get_manager()
|
self.hardware = hardware.get_manager()
|
||||||
self.command_lock = threading.Lock()
|
self.command_lock = threading.Lock()
|
||||||
self.log = log.getLogger(__name__)
|
self.log = log.getLogger(__name__)
|
||||||
@ -213,9 +211,13 @@ class IronicPythonAgent(object):
|
|||||||
"""Run the Ironic Python Agent."""
|
"""Run the Ironic Python Agent."""
|
||||||
self.started_at = _time()
|
self.started_at = _time()
|
||||||
# Get the UUID so we can heartbeat to Ironic
|
# Get the UUID so we can heartbeat to Ironic
|
||||||
self.node = self.api_client.lookup_node(
|
content = self.api_client.lookup_node(
|
||||||
hardware_info=self.hardware.list_hardware_info(),
|
hardware_info=self.hardware.list_hardware_info()
|
||||||
)
|
)
|
||||||
|
if 'node' not in content or 'heartbeat_timeout' not in content:
|
||||||
|
raise LookupError('Lookup return needs node and heartbeat_timeout')
|
||||||
|
self.node = content['node']
|
||||||
|
self.heartbeat_timeout = content['heartbeat_timeout']
|
||||||
self.heartbeater.start()
|
self.heartbeater.start()
|
||||||
wsgi = simple_server.make_server(
|
wsgi = simple_server.make_server(
|
||||||
self.listen_address[0],
|
self.listen_address[0],
|
||||||
|
@ -96,9 +96,11 @@ class APIClient(object):
|
|||||||
+ str(e))
|
+ str(e))
|
||||||
|
|
||||||
if 'node' not in content or 'uuid' not in content['node']:
|
if 'node' not in content or 'uuid' not in content['node']:
|
||||||
raise errors.LookupNodeError('Got invalid data from the API: '
|
raise errors.LookupNodeError('Got invalid node data from the API:'
|
||||||
'{0}'.format(content))
|
' {0}'.format(content))
|
||||||
return content['node']
|
if 'heartbeat_timeout' not in content:
|
||||||
|
raise errors.LookupNodeError('Got invalid')
|
||||||
|
return content
|
||||||
|
|
||||||
def _get_agent_url(self, advertise_address):
|
def _get_agent_url(self, advertise_address):
|
||||||
return 'http://{0}:{1}'.format(advertise_address[0],
|
return 'http://{0}:{1}'.format(advertise_address[0],
|
||||||
|
@ -73,7 +73,7 @@ class TestHeartbeater(unittest.TestCase):
|
|||||||
time_responses.append(50)
|
time_responses.append(50)
|
||||||
|
|
||||||
# SECOND RUN:
|
# SECOND RUN:
|
||||||
# (100 - 50) * .5 = 25 (t becomes ~75)
|
# 50 * .5 = 25
|
||||||
expected_stop_event_calls.append(mock.call(25.0))
|
expected_stop_event_calls.append(mock.call(25.0))
|
||||||
wait_responses.append(False)
|
wait_responses.append(False)
|
||||||
# next heartbeat due at t=180
|
# next heartbeat due at t=180
|
||||||
@ -84,8 +84,8 @@ class TestHeartbeater(unittest.TestCase):
|
|||||||
time_responses.append(80)
|
time_responses.append(80)
|
||||||
|
|
||||||
# THIRD RUN:
|
# THIRD RUN:
|
||||||
# (180 - 80) * .4 = 40 (t becomes ~120)
|
# 50 * .4 = 20
|
||||||
expected_stop_event_calls.append(mock.call(40.0))
|
expected_stop_event_calls.append(mock.call(20.0))
|
||||||
wait_responses.append(False)
|
wait_responses.append(False)
|
||||||
# this heartbeat attempt fails
|
# this heartbeat attempt fails
|
||||||
heartbeat_responses.append(Exception('uh oh!'))
|
heartbeat_responses.append(Exception('uh oh!'))
|
||||||
@ -97,21 +97,23 @@ class TestHeartbeater(unittest.TestCase):
|
|||||||
time_responses.append(125.5)
|
time_responses.append(125.5)
|
||||||
|
|
||||||
# FOURTH RUN:
|
# FOURTH RUN:
|
||||||
# (125.5 - 125.0) * .5 = 0.25
|
# 50 * .5 = 25
|
||||||
expected_stop_event_calls.append(mock.call(0.25))
|
expected_stop_event_calls.append(mock.call(25))
|
||||||
# Stop now
|
# Stop now
|
||||||
wait_responses.append(True)
|
wait_responses.append(True)
|
||||||
|
|
||||||
# Hook it up and run it
|
# Hook it up and run it
|
||||||
mocked_time.side_effect = time_responses
|
mocked_time.side_effect = time_responses
|
||||||
mocked_uniform.side_effect = uniform_responses
|
mocked_uniform.side_effect = uniform_responses
|
||||||
|
self.mock_agent.heartbeat_timeout = 50
|
||||||
self.heartbeater.api.heartbeat.side_effect = heartbeat_responses
|
self.heartbeater.api.heartbeat.side_effect = heartbeat_responses
|
||||||
self.heartbeater.stop_event.wait.side_effect = wait_responses
|
self.heartbeater.stop_event.wait.side_effect = wait_responses
|
||||||
self.heartbeater.run()
|
self.heartbeater.run()
|
||||||
|
|
||||||
# Validate expectations
|
# Validate expectations
|
||||||
self.assertEqual(self.heartbeater.stop_event.wait.call_args_list,
|
self.assertEqual(expected_stop_event_calls,
|
||||||
expected_stop_event_calls)
|
self.heartbeater.stop_event.wait.call_args_list,
|
||||||
|
)
|
||||||
self.assertEqual(self.heartbeater.error_delay, 2.7)
|
self.assertEqual(self.heartbeater.error_delay, 2.7)
|
||||||
|
|
||||||
|
|
||||||
@ -164,6 +166,12 @@ class TestBaseAgent(unittest.TestCase):
|
|||||||
|
|
||||||
self.agent.heartbeater = mock.Mock()
|
self.agent.heartbeater = mock.Mock()
|
||||||
self.agent.api_client.lookup_node = mock.Mock()
|
self.agent.api_client.lookup_node = mock.Mock()
|
||||||
|
self.agent.api_client.lookup_node.return_value = {
|
||||||
|
'node': {
|
||||||
|
'uuid': 'deadbeef-dabb-ad00-b105-f00d00bab10c'
|
||||||
|
},
|
||||||
|
'heartbeat_timeout': 300
|
||||||
|
}
|
||||||
self.agent.run()
|
self.agent.run()
|
||||||
|
|
||||||
listen_addr = ('192.0.2.1', 9999)
|
listen_addr = ('192.0.2.1', 9999)
|
||||||
|
@ -104,7 +104,8 @@ class TestBaseIronicPythonAgent(unittest.TestCase):
|
|||||||
response = httmock.response(status_code=200, content={
|
response = httmock.response(status_code=200, content={
|
||||||
'node': {
|
'node': {
|
||||||
'uuid': 'deadbeef-dabb-ad00-b105-f00d00bab10c'
|
'uuid': 'deadbeef-dabb-ad00-b105-f00d00bab10c'
|
||||||
}
|
},
|
||||||
|
'heartbeat_timeout': 300
|
||||||
})
|
})
|
||||||
|
|
||||||
self.api_client.session.request = mock.Mock()
|
self.api_client.session.request = mock.Mock()
|
||||||
@ -144,10 +145,12 @@ class TestBaseIronicPythonAgent(unittest.TestCase):
|
|||||||
self.assertRaises(errors.LookupNodeError,
|
self.assertRaises(errors.LookupNodeError,
|
||||||
self.api_client.lookup_node,
|
self.api_client.lookup_node,
|
||||||
hardware_info=self.hardware_info,
|
hardware_info=self.hardware_info,
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_lookup_node_bad_response_data(self):
|
def test_lookup_node_bad_response_data(self):
|
||||||
response = httmock.response(status_code=200, content='a')
|
response = httmock.response(status_code=200, content={
|
||||||
|
'heartbeat_timeout': 300
|
||||||
|
})
|
||||||
|
|
||||||
self.api_client.session.request = mock.Mock()
|
self.api_client.session.request = mock.Mock()
|
||||||
self.api_client.session.request.return_value = response
|
self.api_client.session.request.return_value = response
|
||||||
@ -155,7 +158,22 @@ class TestBaseIronicPythonAgent(unittest.TestCase):
|
|||||||
self.assertRaises(errors.LookupNodeError,
|
self.assertRaises(errors.LookupNodeError,
|
||||||
self.api_client.lookup_node,
|
self.api_client.lookup_node,
|
||||||
hardware_info=self.hardware_info
|
hardware_info=self.hardware_info
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_lookup_node_no_heartbeat_timeout(self):
|
||||||
|
response = httmock.response(status_code=200, content={
|
||||||
|
'node': {
|
||||||
|
'uuid': 'deadbeef-dabb-ad00-b105-f00d00bab10c'
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
self.api_client.session.request = mock.Mock()
|
||||||
|
self.api_client.session.request.return_value = response
|
||||||
|
|
||||||
|
self.assertRaises(errors.LookupNodeError,
|
||||||
|
self.api_client.lookup_node,
|
||||||
|
hardware_info=self.hardware_info
|
||||||
|
)
|
||||||
|
|
||||||
def test_lookup_node_bad_response_body(self):
|
def test_lookup_node_bad_response_body(self):
|
||||||
response = httmock.response(status_code=200, content={
|
response = httmock.response(status_code=200, content={
|
||||||
|
Loading…
x
Reference in New Issue
Block a user