Wait for power off before finishing discovery
This used to be part of returning-to-ramdisk, but was lost after revert. Also do not make the ramdisk wait for it's own power off. Change-Id: Ia43f834023b189324cba38a19cc782fc0b7745d4 Closes-Bug: #1401872
This commit is contained in:
parent
bdaf0f4d15
commit
6432b534f6
|
@ -29,6 +29,7 @@ from ironic_discoverd import utils
|
||||||
LOG = logging.getLogger("ironic_discoverd.process")
|
LOG = logging.getLogger("ironic_discoverd.process")
|
||||||
|
|
||||||
_POWER_CHECK_PERIOD = 5
|
_POWER_CHECK_PERIOD = 5
|
||||||
|
_POWER_OFF_CHECK_PERIOD = 5
|
||||||
|
|
||||||
|
|
||||||
def process(node_info):
|
def process(node_info):
|
||||||
|
@ -110,7 +111,7 @@ def _process_node(ironic, node, node_info, cached_node):
|
||||||
'ipmi_username': node.driver_info.get('ipmi_username'),
|
'ipmi_username': node.driver_info.get('ipmi_username'),
|
||||||
'ipmi_password': node.driver_info.get('ipmi_password')}
|
'ipmi_password': node.driver_info.get('ipmi_password')}
|
||||||
else:
|
else:
|
||||||
_finish_discovery(ironic, cached_node)
|
eventlet.greenthread.spawn_n(_finish_discovery, ironic, cached_node)
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
@ -131,21 +132,38 @@ def _wait_for_power_management(ironic, cached_node):
|
||||||
'after discovery', cached_node.uuid)
|
'after discovery', cached_node.uuid)
|
||||||
|
|
||||||
|
|
||||||
def _force_power_off(ironic, node):
|
def _force_power_off(ironic, cached_node):
|
||||||
LOG.debug('Forcing power off of node %s', node.uuid)
|
LOG.debug('Forcing power off of node %s', cached_node.uuid)
|
||||||
try:
|
try:
|
||||||
utils.retry_on_conflict(ironic.node.set_power_state, node.uuid, 'off')
|
utils.retry_on_conflict(ironic.node.set_power_state,
|
||||||
|
cached_node.uuid, 'off')
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
LOG.error('Failed to power off node %s, check it\'s power '
|
LOG.error('Failed to power off node %s, check it\'s power '
|
||||||
'management configuration:\n%s', node.uuid, exc)
|
'management configuration:\n%s', cached_node.uuid, exc)
|
||||||
raise utils.DiscoveryFailed('Failed to power off node %s' % node.uuid)
|
raise utils.DiscoveryFailed('Failed to power off node %s'
|
||||||
|
% cached_node.uuid)
|
||||||
|
|
||||||
|
deadline = cached_node.started_at + conf.getint('discoverd', 'timeout')
|
||||||
|
while time.time() < deadline:
|
||||||
|
node = ironic.node.get(cached_node.uuid)
|
||||||
|
if (node.power_state or '').lower() == 'power off':
|
||||||
|
return
|
||||||
|
LOG.info('Waiting for node %s to power off, current state is %s',
|
||||||
|
cached_node.uuid, node.power_state)
|
||||||
|
eventlet.greenthread.sleep(_POWER_OFF_CHECK_PERIOD)
|
||||||
|
|
||||||
|
LOG.error('Timeout waiting for node %s to power off after discovery',
|
||||||
|
cached_node.uuid)
|
||||||
|
raise utils.DiscoveryFailed(
|
||||||
|
'Timeout waiting for node %s to power off after discovery',
|
||||||
|
cached_node.uuid)
|
||||||
|
|
||||||
|
|
||||||
def _finish_discovery(ironic, node):
|
def _finish_discovery(ironic, cached_node):
|
||||||
_force_power_off(ironic, node)
|
_force_power_off(ironic, cached_node)
|
||||||
|
|
||||||
patch = [{'op': 'add', 'path': '/extra/newly_discovered', 'value': 'true'},
|
patch = [{'op': 'add', 'path': '/extra/newly_discovered', 'value': 'true'},
|
||||||
{'op': 'remove', 'path': '/extra/on_discovery'}]
|
{'op': 'remove', 'path': '/extra/on_discovery'}]
|
||||||
utils.retry_on_conflict(ironic.node.update, node.uuid, patch)
|
utils.retry_on_conflict(ironic.node.update, cached_node.uuid, patch)
|
||||||
|
|
||||||
LOG.info('Discovery finished successfully for node %s', node.uuid)
|
LOG.info('Discovery finished successfully for node %s', cached_node.uuid)
|
||||||
|
|
|
@ -202,6 +202,7 @@ class TestProcessNode(BaseTest):
|
||||||
conf.CONF.set('discoverd', 'processing_hooks',
|
conf.CONF.set('discoverd', 'processing_hooks',
|
||||||
'ramdisk_error,scheduler,validate_interfaces,example')
|
'ramdisk_error,scheduler,validate_interfaces,example')
|
||||||
self.validate_attempts = 5
|
self.validate_attempts = 5
|
||||||
|
self.power_off_attempts = 2
|
||||||
self.data['macs'] = self.macs # validate_interfaces hook
|
self.data['macs'] = self.macs # validate_interfaces hook
|
||||||
self.cached_node = node_cache.NodeInfo(uuid=self.uuid,
|
self.cached_node = node_cache.NodeInfo(uuid=self.uuid,
|
||||||
started_at=self.started_at)
|
started_at=self.started_at)
|
||||||
|
@ -218,6 +219,10 @@ class TestProcessNode(BaseTest):
|
||||||
self.cli.node.validate.side_effect = self.fake_validate()
|
self.cli.node.validate.side_effect = self.fake_validate()
|
||||||
self.cli.port.create.side_effect = self.ports
|
self.cli.port.create.side_effect = self.ports
|
||||||
self.cli.node.update.return_value = self.node
|
self.cli.node.update.return_value = self.node
|
||||||
|
# Simulate longer power off
|
||||||
|
self.cli.node.get.side_effect = (
|
||||||
|
[self.node] * self.power_off_attempts
|
||||||
|
+ [mock.Mock(power_state='power off')])
|
||||||
|
|
||||||
def fake_validate(self):
|
def fake_validate(self):
|
||||||
# Simulate long ramdisk task
|
# Simulate long ramdisk task
|
||||||
|
@ -240,6 +245,8 @@ class TestProcessNode(BaseTest):
|
||||||
self.cli.node.update.assert_any_call(self.uuid, self.patch_after)
|
self.cli.node.update.assert_any_call(self.uuid, self.patch_after)
|
||||||
self.cli.node.set_power_state.assert_called_once_with(self.uuid, 'off')
|
self.cli.node.set_power_state.assert_called_once_with(self.uuid, 'off')
|
||||||
self.assertFalse(self.cli.node.validate.called)
|
self.assertFalse(self.cli.node.validate.called)
|
||||||
|
self.assertEqual(self.power_off_attempts + 1,
|
||||||
|
self.cli.node.get.call_count)
|
||||||
|
|
||||||
post_hook_mock.assert_called_once_with(self.node, mock.ANY,
|
post_hook_mock.assert_called_once_with(self.node, mock.ANY,
|
||||||
self.data)
|
self.data)
|
||||||
|
@ -276,6 +283,18 @@ class TestProcessNode(BaseTest):
|
||||||
self.cli.node.set_power_state.assert_called_with(self.uuid, 'off')
|
self.cli.node.set_power_state.assert_called_with(self.uuid, 'off')
|
||||||
self.assertEqual(2, self.cli.node.set_power_state.call_count)
|
self.assertEqual(2, self.cli.node.set_power_state.call_count)
|
||||||
|
|
||||||
|
@mock.patch.object(time, 'time')
|
||||||
|
def test_power_off_timeout(self, time_mock, filters_mock,
|
||||||
|
post_hook_mock):
|
||||||
|
conf.CONF.set('discoverd', 'timeout', '100')
|
||||||
|
time_mock.return_value = self.started_at + 1000
|
||||||
|
self.cli.node.get.return_value = self.node
|
||||||
|
|
||||||
|
self.assertRaisesRegexp(utils.DiscoveryFailed, 'power off', self.call)
|
||||||
|
|
||||||
|
self.cli.node.update.assert_called_once_with(self.uuid,
|
||||||
|
self.patch_before)
|
||||||
|
|
||||||
def test_port_failed(self, filters_mock, post_hook_mock):
|
def test_port_failed(self, filters_mock, post_hook_mock):
|
||||||
self.ports[0] = exceptions.Conflict()
|
self.ports[0] = exceptions.Conflict()
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue