ironic/ironic/tests/unit/conductor/test_cleaning.py

1058 lines
48 KiB
Python

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tests for cleaning bits."""
import mock
from oslo_config import cfg
from oslo_utils import uuidutils
from ironic.common import exception
from ironic.common import states
from ironic.conductor import cleaning
from ironic.conductor import steps as conductor_steps
from ironic.conductor import task_manager
from ironic.drivers.modules import fake
from ironic.drivers.modules.network import flat as n_flat
from ironic.tests.unit.db import base as db_base
from ironic.tests.unit.objects import utils as obj_utils
CONF = cfg.CONF
class DoNodeCleanTestCase(db_base.DbTestCase):
def setUp(self):
super(DoNodeCleanTestCase, self).setUp()
self.config(automated_clean=True, group='conductor')
self.power_update = {
'step': 'update_firmware', 'priority': 10, 'interface': 'power'}
self.deploy_update = {
'step': 'update_firmware', 'priority': 10, 'interface': 'deploy'}
self.deploy_erase = {
'step': 'erase_disks', 'priority': 20, 'interface': 'deploy'}
# Automated cleaning should be executed in this order
self.clean_steps = [self.deploy_erase, self.power_update,
self.deploy_update]
self.next_clean_step_index = 1
# Manual clean step
self.deploy_raid = {
'step': 'build_raid', 'priority': 0, 'interface': 'deploy'}
def __do_node_clean_validate_fail(self, mock_validate, clean_steps=None):
# InvalidParameterValue should cause node to go to CLEANFAIL
mock_validate.side_effect = exception.InvalidParameterValue('error')
tgt_prov_state = states.MANAGEABLE if clean_steps else states.AVAILABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state)
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task, clean_steps=clean_steps)
node.refresh()
self.assertEqual(states.CLEANFAIL, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
autospec=True)
def test__do_node_clean_automated_power_validate_fail(self, mock_validate):
self.__do_node_clean_validate_fail(mock_validate)
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
autospec=True)
def test__do_node_clean_manual_power_validate_fail(self, mock_validate):
self.__do_node_clean_validate_fail(mock_validate, clean_steps=[])
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
autospec=True)
def test__do_node_clean_automated_network_validate_fail(self,
mock_validate):
self.__do_node_clean_validate_fail(mock_validate)
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
autospec=True)
def test__do_node_clean_manual_network_validate_fail(self, mock_validate):
self.__do_node_clean_validate_fail(mock_validate, clean_steps=[])
@mock.patch.object(cleaning, 'LOG', autospec=True)
@mock.patch.object(conductor_steps, 'set_node_cleaning_steps',
autospec=True)
@mock.patch.object(cleaning, 'do_next_clean_step', autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_cleaning',
autospec=True)
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakeBIOS.cache_bios_settings',
autospec=True)
def _test__do_node_clean_cache_bios(self, mock_bios, mock_validate,
mock_prep, mock_next_step, mock_steps,
mock_log, clean_steps=None,
enable_unsupported=False,
enable_exception=False):
if enable_unsupported:
mock_bios.side_effect = exception.UnsupportedDriverExtension('')
elif enable_exception:
mock_bios.side_effect = exception.IronicException('test')
mock_prep.return_value = states.NOSTATE
tgt_prov_state = states.MANAGEABLE if clean_steps else states.AVAILABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state)
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task, clean_steps=clean_steps)
node.refresh()
mock_bios.assert_called_once_with(mock.ANY, task)
if clean_steps:
self.assertEqual(states.CLEANING, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
else:
self.assertEqual(states.CLEANING, node.provision_state)
self.assertEqual(states.AVAILABLE, node.target_provision_state)
mock_validate.assert_called_once_with(mock.ANY, task)
if enable_exception:
mock_log.exception.assert_called_once_with(
'Caching of bios settings failed on node {}. '
'Continuing with node cleaning.'
.format(node.uuid))
def test__do_node_clean_manual_cache_bios(self):
self._test__do_node_clean_cache_bios(clean_steps=[self.deploy_raid])
def test__do_node_clean_automated_cache_bios(self):
self._test__do_node_clean_cache_bios()
def test__do_node_clean_manual_cache_bios_exception(self):
self._test__do_node_clean_cache_bios(clean_steps=[self.deploy_raid],
enable_exception=True)
def test__do_node_clean_automated_cache_bios_exception(self):
self._test__do_node_clean_cache_bios(enable_exception=True)
def test__do_node_clean_manual_cache_bios_unsupported(self):
self._test__do_node_clean_cache_bios(clean_steps=[self.deploy_raid],
enable_unsupported=True)
def test__do_node_clean_automated_cache_bios_unsupported(self):
self._test__do_node_clean_cache_bios(enable_unsupported=True)
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
autospec=True)
def test__do_node_clean_automated_disabled(self, mock_validate):
self.config(automated_clean=False, group='conductor')
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=states.AVAILABLE,
last_error=None)
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task)
node.refresh()
# Assert that the node was moved to available without cleaning
self.assertFalse(mock_validate.called)
self.assertEqual(states.AVAILABLE, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_steps', node.driver_internal_info)
self.assertNotIn('clean_step_index', node.driver_internal_info)
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
autospec=True)
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
autospec=True)
def test__do_node_clean_automated_disabled_individual_enabled(
self, mock_network, mock_validate):
self.config(automated_clean=False, group='conductor')
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=states.AVAILABLE,
last_error=None, automated_clean=True)
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task)
node.refresh()
# Assert that the node clean was called
self.assertTrue(mock_validate.called)
self.assertIn('clean_steps', node.driver_internal_info)
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
autospec=True)
def test__do_node_clean_automated_disabled_individual_disabled(
self, mock_validate):
self.config(automated_clean=False, group='conductor')
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=states.AVAILABLE,
last_error=None, automated_clean=False)
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task)
node.refresh()
# Assert that the node was moved to available without cleaning
self.assertFalse(mock_validate.called)
self.assertEqual(states.AVAILABLE, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_steps', node.driver_internal_info)
self.assertNotIn('clean_step_index', node.driver_internal_info)
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
autospec=True)
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
autospec=True)
def test__do_node_clean_automated_enabled(self, mock_validate,
mock_network):
self.config(automated_clean=True, group='conductor')
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=states.AVAILABLE,
last_error=None,
driver_internal_info={'agent_url': 'url'})
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task)
node.refresh()
# Assert that the node was cleaned
self.assertTrue(mock_validate.called)
self.assertIn('clean_steps', node.driver_internal_info)
self.assertNotIn('agent_url', node.driver_internal_info)
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
autospec=True)
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
autospec=True)
def test__do_node_clean_automated_enabled_individual_enabled(
self, mock_network, mock_validate):
self.config(automated_clean=True, group='conductor')
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=states.AVAILABLE,
last_error=None, automated_clean=True)
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task)
node.refresh()
# Assert that the node was cleaned
self.assertTrue(mock_validate.called)
self.assertIn('clean_steps', node.driver_internal_info)
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
autospec=True)
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
autospec=True)
def test__do_node_clean_automated_enabled_individual_none(
self, mock_validate, mock_network):
self.config(automated_clean=True, group='conductor')
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=states.AVAILABLE,
last_error=None, automated_clean=None)
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task)
node.refresh()
# Assert that the node was cleaned
self.assertTrue(mock_validate.called)
self.assertIn('clean_steps', node.driver_internal_info)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.tear_down_cleaning',
autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_cleaning',
autospec=True)
def test__do_node_clean_maintenance(self, mock_prep, mock_tear_down):
CONF.set_override('allow_provisioning_in_maintenance', False,
group='conductor')
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=states.AVAILABLE,
maintenance=True,
maintenance_reason='Original reason')
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task)
node.refresh()
self.assertEqual(states.CLEANFAIL, node.provision_state)
self.assertEqual(states.AVAILABLE, node.target_provision_state)
self.assertIn('is not allowed', node.last_error)
self.assertTrue(node.maintenance)
self.assertEqual('Original reason', node.maintenance_reason)
self.assertFalse(mock_prep.called)
self.assertFalse(mock_tear_down.called)
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_cleaning',
autospec=True)
def __do_node_clean_prepare_clean_fail(self, mock_prep, mock_validate,
clean_steps=None):
# Exception from task.driver.deploy.prepare_cleaning should cause node
# to go to CLEANFAIL
mock_prep.side_effect = exception.InvalidParameterValue('error')
tgt_prov_state = states.MANAGEABLE if clean_steps else states.AVAILABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state)
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task, clean_steps=clean_steps)
node.refresh()
self.assertEqual(states.CLEANFAIL, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
mock_prep.assert_called_once_with(mock.ANY, task)
mock_validate.assert_called_once_with(mock.ANY, task)
def test__do_node_clean_automated_prepare_clean_fail(self):
self.__do_node_clean_prepare_clean_fail()
def test__do_node_clean_manual_prepare_clean_fail(self):
self.__do_node_clean_prepare_clean_fail(clean_steps=[self.deploy_raid])
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_cleaning',
autospec=True)
def __do_node_clean_prepare_clean_wait(self, mock_prep, mock_validate,
clean_steps=None):
mock_prep.return_value = states.CLEANWAIT
tgt_prov_state = states.MANAGEABLE if clean_steps else states.AVAILABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state)
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task, clean_steps=clean_steps)
node.refresh()
self.assertEqual(states.CLEANWAIT, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
mock_prep.assert_called_once_with(mock.ANY, mock.ANY)
mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
def test__do_node_clean_automated_prepare_clean_wait(self):
self.__do_node_clean_prepare_clean_wait()
def test__do_node_clean_manual_prepare_clean_wait(self):
self.__do_node_clean_prepare_clean_wait(clean_steps=[self.deploy_raid])
@mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
@mock.patch.object(conductor_steps, 'set_node_cleaning_steps',
autospec=True)
def __do_node_clean_steps_fail(self, mock_steps, mock_validate,
clean_steps=None, invalid_exc=True):
if invalid_exc:
mock_steps.side_effect = exception.InvalidParameterValue('invalid')
else:
mock_steps.side_effect = exception.NodeCleaningFailure('failure')
tgt_prov_state = states.MANAGEABLE if clean_steps else states.AVAILABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
uuid=uuidutils.generate_uuid(),
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state)
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task, clean_steps=clean_steps)
mock_validate.assert_called_once_with(mock.ANY, task)
node.refresh()
self.assertEqual(states.CLEANFAIL, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
mock_steps.assert_called_once_with(mock.ANY)
def test__do_node_clean_automated_steps_fail(self):
for invalid in (True, False):
self.__do_node_clean_steps_fail(invalid_exc=invalid)
def test__do_node_clean_manual_steps_fail(self):
for invalid in (True, False):
self.__do_node_clean_steps_fail(clean_steps=[self.deploy_raid],
invalid_exc=invalid)
@mock.patch.object(conductor_steps, 'set_node_cleaning_steps',
autospec=True)
@mock.patch.object(cleaning, 'do_next_clean_step', autospec=True)
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
autospec=True)
def __do_node_clean(self, mock_power_valid, mock_network_valid,
mock_next_step, mock_steps, clean_steps=None):
if clean_steps:
tgt_prov_state = states.MANAGEABLE
driver_info = {}
else:
tgt_prov_state = states.AVAILABLE
driver_info = {'clean_steps': self.clean_steps}
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
power_state=states.POWER_OFF,
driver_internal_info=driver_info)
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_node_clean(task, clean_steps=clean_steps)
node.refresh()
mock_power_valid.assert_called_once_with(mock.ANY, task)
mock_network_valid.assert_called_once_with(mock.ANY, task)
mock_next_step.assert_called_once_with(task, 0)
mock_steps.assert_called_once_with(task)
if clean_steps:
self.assertEqual(clean_steps,
node.driver_internal_info['clean_steps'])
# Check that state didn't change
self.assertEqual(states.CLEANING, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
def test__do_node_clean_automated(self):
self.__do_node_clean()
def test__do_node_clean_manual(self):
self.__do_node_clean(clean_steps=[self.deploy_raid])
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
autospec=True)
def _do_next_clean_step_first_step_async(self, return_state, mock_execute,
clean_steps=None):
# Execute the first async clean step on a node
driver_internal_info = {'clean_step_index': None}
if clean_steps:
tgt_prov_state = states.MANAGEABLE
driver_internal_info['clean_steps'] = clean_steps
else:
tgt_prov_state = states.AVAILABLE
driver_internal_info['clean_steps'] = self.clean_steps
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info=driver_internal_info,
clean_step={})
mock_execute.return_value = return_state
expected_first_step = node.driver_internal_info['clean_steps'][0]
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_next_clean_step(task, 0)
node.refresh()
self.assertEqual(states.CLEANWAIT, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual(expected_first_step, node.clean_step)
self.assertEqual(0, node.driver_internal_info['clean_step_index'])
mock_execute.assert_called_once_with(
mock.ANY, mock.ANY, expected_first_step)
def test_do_next_clean_step_automated_first_step_async(self):
self._do_next_clean_step_first_step_async(states.CLEANWAIT)
def test_do_next_clean_step_manual_first_step_async(self):
self._do_next_clean_step_first_step_async(
states.CLEANWAIT, clean_steps=[self.deploy_raid])
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_clean_step',
autospec=True)
def _do_next_clean_step_continue_from_last_cleaning(self, return_state,
mock_execute,
manual=False):
# Resume an in-progress cleaning after the first async step
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={'clean_steps': self.clean_steps,
'clean_step_index': 0},
clean_step=self.clean_steps[0])
mock_execute.return_value = return_state
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_next_clean_step(task, self.next_clean_step_index)
node.refresh()
self.assertEqual(states.CLEANWAIT, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual(self.clean_steps[1], node.clean_step)
self.assertEqual(1, node.driver_internal_info['clean_step_index'])
mock_execute.assert_called_once_with(
mock.ANY, mock.ANY, self.clean_steps[1])
def test_do_next_clean_step_continue_from_last_cleaning(self):
self._do_next_clean_step_continue_from_last_cleaning(states.CLEANWAIT)
def test_do_next_clean_step_manual_continue_from_last_cleaning(self):
self._do_next_clean_step_continue_from_last_cleaning(states.CLEANWAIT,
manual=True)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
autospec=True)
def _do_next_clean_step_last_step_noop(self, mock_execute, manual=False,
retired=False, fast_track=False):
if fast_track:
self.config(fast_track=True, group='deploy')
# Resume where last_step is the last cleaning step, should be noop
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
info = {'clean_steps': self.clean_steps,
'clean_step_index': len(self.clean_steps) - 1,
'agent_url': 'test-url',
'agent_secret_token': 'token'}
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info=info,
clean_step=self.clean_steps[-1],
retired=retired)
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_next_clean_step(task, None)
node.refresh()
# retired nodes move to manageable upon cleaning
if retired:
tgt_prov_state = states.MANAGEABLE
# Cleaning should be complete without calling additional steps
self.assertEqual(tgt_prov_state, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertIsNone(node.driver_internal_info['clean_steps'])
self.assertFalse(mock_execute.called)
if fast_track:
self.assertEqual('test-url',
node.driver_internal_info.get('agent_url'))
self.assertIsNotNone(
node.driver_internal_info.get('agent_secret_token'))
else:
self.assertNotIn('agent_url', node.driver_internal_info)
self.assertNotIn('agent_secret_token',
node.driver_internal_info)
def test__do_next_clean_step_automated_last_step_noop(self):
self._do_next_clean_step_last_step_noop()
def test__do_next_clean_step_manual_last_step_noop(self):
self._do_next_clean_step_last_step_noop(manual=True)
def test__do_next_clean_step_retired_last_step_change_tgt_state(self):
self._do_next_clean_step_last_step_noop(retired=True)
def test__do_next_clean_step_last_step_noop_fast_track(self):
self._do_next_clean_step_last_step_noop(fast_track=True)
@mock.patch('ironic.drivers.utils.collect_ramdisk_logs', autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_clean_step',
autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
autospec=True)
def _do_next_clean_step_all(self, mock_deploy_execute,
mock_power_execute, mock_collect_logs,
manual=False):
# Run all steps from start to finish (all synchronous)
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={'clean_steps': self.clean_steps,
'clean_step_index': None},
clean_step={})
def fake_deploy(conductor_obj, task, step):
driver_internal_info = task.node.driver_internal_info
driver_internal_info['goober'] = 'test'
task.node.driver_internal_info = driver_internal_info
task.node.save()
mock_deploy_execute.side_effect = fake_deploy
mock_power_execute.return_value = None
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_next_clean_step(task, 0)
node.refresh()
# Cleaning should be complete
self.assertEqual(tgt_prov_state, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertEqual('test', node.driver_internal_info['goober'])
self.assertIsNone(node.driver_internal_info['clean_steps'])
mock_power_execute.assert_called_once_with(mock.ANY, mock.ANY,
self.clean_steps[1])
mock_deploy_execute.assert_has_calls(
[mock.call(mock.ANY, mock.ANY, self.clean_steps[0]),
mock.call(mock.ANY, mock.ANY, self.clean_steps[2])])
self.assertFalse(mock_collect_logs.called)
def test_do_next_clean_step_automated_all(self):
self._do_next_clean_step_all()
def test_do_next_clean_step_manual_all(self):
self._do_next_clean_step_all(manual=True)
@mock.patch('ironic.drivers.utils.collect_ramdisk_logs', autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_clean_step',
autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
autospec=True)
def test_do_next_clean_step_collect_logs(self, mock_deploy_execute,
mock_power_execute,
mock_collect_logs):
CONF.set_override('deploy_logs_collect', 'always', group='agent')
# Run all steps from start to finish (all synchronous)
tgt_prov_state = states.MANAGEABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={'clean_steps': self.clean_steps,
'clean_step_index': None},
clean_step={})
def fake_deploy(conductor_obj, task, step):
driver_internal_info = task.node.driver_internal_info
driver_internal_info['goober'] = 'test'
task.node.driver_internal_info = driver_internal_info
task.node.save()
mock_deploy_execute.side_effect = fake_deploy
mock_power_execute.return_value = None
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_next_clean_step(task, 0)
node.refresh()
# Cleaning should be complete
self.assertEqual(tgt_prov_state, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertEqual('test', node.driver_internal_info['goober'])
self.assertIsNone(node.driver_internal_info['clean_steps'])
mock_power_execute.assert_called_once_with(mock.ANY, mock.ANY,
self.clean_steps[1])
mock_deploy_execute.assert_has_calls(
[mock.call(mock.ANY, mock.ANY, self.clean_steps[0]),
mock.call(mock.ANY, mock.ANY, self.clean_steps[2])])
mock_collect_logs.assert_called_once_with(mock.ANY, label='cleaning')
@mock.patch('ironic.drivers.utils.collect_ramdisk_logs', autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
autospec=True)
@mock.patch.object(fake.FakeDeploy, 'tear_down_cleaning', autospec=True)
def _do_next_clean_step_execute_fail(self, tear_mock, mock_execute,
mock_collect_logs, manual=False):
# When a clean step fails, go to CLEANFAIL
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={'clean_steps': self.clean_steps,
'clean_step_index': None},
clean_step={})
mock_execute.side_effect = Exception()
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_next_clean_step(task, 0)
tear_mock.assert_called_once_with(task.driver.deploy, task)
node.refresh()
# Make sure we go to CLEANFAIL, clear clean_steps
self.assertEqual(states.CLEANFAIL, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertIsNotNone(node.last_error)
self.assertTrue(node.maintenance)
mock_execute.assert_called_once_with(
mock.ANY, mock.ANY, self.clean_steps[0])
mock_collect_logs.assert_called_once_with(mock.ANY, label='cleaning')
def test__do_next_clean_step_automated_execute_fail(self):
self._do_next_clean_step_execute_fail()
def test__do_next_clean_step_manual_execute_fail(self):
self._do_next_clean_step_execute_fail(manual=True)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
autospec=True)
def test_do_next_clean_step_oob_reboot(self, mock_execute):
# When a clean step fails, go to CLEANWAIT
tgt_prov_state = states.MANAGEABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={'clean_steps': self.clean_steps,
'clean_step_index': None,
'cleaning_reboot': True},
clean_step={})
mock_execute.side_effect = exception.AgentConnectionFailed(
reason='failed')
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_next_clean_step(task, 0)
node.refresh()
# Make sure we go to CLEANWAIT
self.assertEqual(states.CLEANWAIT, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual(self.clean_steps[0], node.clean_step)
self.assertEqual(0, node.driver_internal_info['clean_step_index'])
self.assertFalse(node.driver_internal_info['skip_current_clean_step'])
mock_execute.assert_called_once_with(
mock.ANY, mock.ANY, self.clean_steps[0])
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
autospec=True)
def test_do_next_clean_step_oob_reboot_last_step(self, mock_execute):
# Resume where last_step is the last cleaning step
tgt_prov_state = states.MANAGEABLE
info = {'clean_steps': self.clean_steps,
'cleaning_reboot': True,
'clean_step_index': len(self.clean_steps) - 1}
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info=info,
clean_step=self.clean_steps[-1])
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_next_clean_step(task, None)
node.refresh()
# Cleaning should be complete without calling additional steps
self.assertEqual(tgt_prov_state, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertNotIn('cleaning_reboot', node.driver_internal_info)
self.assertIsNone(node.driver_internal_info['clean_steps'])
self.assertFalse(mock_execute.called)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
autospec=True)
@mock.patch.object(fake.FakeDeploy, 'tear_down_cleaning', autospec=True)
def test_do_next_clean_step_oob_reboot_fail(self, tear_mock,
mock_execute):
# When a clean step fails with no reboot requested go to CLEANFAIL
tgt_prov_state = states.MANAGEABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={'clean_steps': self.clean_steps,
'clean_step_index': None},
clean_step={})
mock_execute.side_effect = exception.AgentConnectionFailed(
reason='failed')
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_next_clean_step(task, 0)
tear_mock.assert_called_once_with(task.driver.deploy, task)
node.refresh()
# Make sure we go to CLEANFAIL, clear clean_steps
self.assertEqual(states.CLEANFAIL, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertNotIn('skip_current_clean_step', node.driver_internal_info)
self.assertIsNotNone(node.last_error)
self.assertTrue(node.maintenance)
mock_execute.assert_called_once_with(
mock.ANY, mock.ANY, self.clean_steps[0])
@mock.patch.object(cleaning, 'LOG', autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_clean_step',
autospec=True)
@mock.patch.object(fake.FakeDeploy, 'tear_down_cleaning', autospec=True)
def _do_next_clean_step_fail_in_tear_down_cleaning(
self, tear_mock, power_exec_mock, deploy_exec_mock, log_mock,
manual=True):
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={'clean_steps': self.clean_steps,
'clean_step_index': None},
clean_step={})
deploy_exec_mock.return_value = None
power_exec_mock.return_value = None
tear_mock.side_effect = Exception('boom')
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_next_clean_step(task, 0)
node.refresh()
# Make sure we go to CLEANFAIL, clear clean_steps
self.assertEqual(states.CLEANFAIL, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertIsNotNone(node.last_error)
self.assertEqual(1, tear_mock.call_count)
self.assertTrue(node.maintenance)
deploy_exec_calls = [
mock.call(mock.ANY, mock.ANY, self.clean_steps[0]),
mock.call(mock.ANY, mock.ANY, self.clean_steps[2]),
]
self.assertEqual(deploy_exec_calls, deploy_exec_mock.call_args_list)
power_exec_calls = [
mock.call(mock.ANY, mock.ANY, self.clean_steps[1]),
]
self.assertEqual(power_exec_calls, power_exec_mock.call_args_list)
log_mock.exception.assert_called_once_with(
'Failed to tear down from cleaning for node {}, reason: boom'
.format(node.uuid))
def test__do_next_clean_step_automated_fail_in_tear_down_cleaning(self):
self._do_next_clean_step_fail_in_tear_down_cleaning()
def test__do_next_clean_step_manual_fail_in_tear_down_cleaning(self):
self._do_next_clean_step_fail_in_tear_down_cleaning(manual=True)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
autospec=True)
def _do_next_clean_step_no_steps(self, mock_execute, manual=False,
fast_track=False):
if fast_track:
self.config(fast_track=True, group='deploy')
for info in ({'clean_steps': None, 'clean_step_index': None,
'agent_url': 'test-url', 'agent_secret_token': 'magic'},
{'clean_steps': None, 'agent_url': 'test-url',
'agent_secret_token': 'it_is_a_kind_of_magic'}):
# Resume where there are no steps, should be a noop
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
uuid=uuidutils.generate_uuid(),
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info=info,
clean_step={})
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_next_clean_step(task, None)
node.refresh()
# Cleaning should be complete without calling additional steps
self.assertEqual(tgt_prov_state, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertFalse(mock_execute.called)
if fast_track:
self.assertEqual('test-url',
node.driver_internal_info.get('agent_url'))
self.assertIsNotNone(
node.driver_internal_info.get('agent_secret_token'))
else:
self.assertNotIn('agent_url', node.driver_internal_info)
self.assertNotIn('agent_secret_token',
node.driver_internal_info)
mock_execute.reset_mock()
def test__do_next_clean_step_automated_no_steps(self):
self._do_next_clean_step_no_steps()
def test__do_next_clean_step_manual_no_steps(self):
self._do_next_clean_step_no_steps(manual=True)
def test__do_next_clean_step_fast_track(self):
self._do_next_clean_step_no_steps(fast_track=True)
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_clean_step',
autospec=True)
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
autospec=True)
def _do_next_clean_step_bad_step_return_value(
self, deploy_exec_mock, power_exec_mock, manual=False):
# When a clean step fails, go to CLEANFAIL
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={'clean_steps': self.clean_steps,
'clean_step_index': None},
clean_step={})
deploy_exec_mock.return_value = "foo"
with task_manager.acquire(
self.context, node.uuid, shared=False) as task:
cleaning.do_next_clean_step(task, 0)
node.refresh()
# Make sure we go to CLEANFAIL, clear clean_steps
self.assertEqual(states.CLEANFAIL, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertIsNotNone(node.last_error)
self.assertTrue(node.maintenance)
deploy_exec_mock.assert_called_once_with(mock.ANY, mock.ANY,
self.clean_steps[0])
# Make sure we don't execute any other step and return
self.assertFalse(power_exec_mock.called)
def test__do_next_clean_step_automated_bad_step_return_value(self):
self._do_next_clean_step_bad_step_return_value()
def test__do_next_clean_step_manual_bad_step_return_value(self):
self._do_next_clean_step_bad_step_return_value(manual=True)
class DoNodeCleanAbortTestCase(db_base.DbTestCase):
@mock.patch.object(fake.FakeDeploy, 'tear_down_cleaning', autospec=True)
def _test__do_node_clean_abort(self, step_name, tear_mock):
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANFAIL,
target_provision_state=states.AVAILABLE,
clean_step={'step': 'foo', 'abortable': True},
driver_internal_info={
'agent_url': 'some url',
'agent_secret_token': 'token',
'clean_step_index': 2,
'cleaning_reboot': True,
'cleaning_polling': True,
'skip_current_clean_step': True})
with task_manager.acquire(self.context, node.uuid) as task:
cleaning.do_node_clean_abort(task, step_name=step_name)
self.assertIsNotNone(task.node.last_error)
tear_mock.assert_called_once_with(task.driver.deploy, task)
if step_name:
self.assertIn(step_name, task.node.last_error)
# assert node's clean_step and metadata was cleaned up
self.assertEqual({}, task.node.clean_step)
self.assertNotIn('clean_step_index',
task.node.driver_internal_info)
self.assertNotIn('cleaning_reboot',
task.node.driver_internal_info)
self.assertNotIn('cleaning_polling',
task.node.driver_internal_info)
self.assertNotIn('skip_current_clean_step',
task.node.driver_internal_info)
self.assertNotIn('agent_url',
task.node.driver_internal_info)
self.assertNotIn('agent_secret_token',
task.node.driver_internal_info)
def test__do_node_clean_abort(self):
self._test__do_node_clean_abort(None)
def test__do_node_clean_abort_with_step_name(self):
self._test__do_node_clean_abort('foo')
@mock.patch.object(fake.FakeDeploy, 'tear_down_cleaning', autospec=True)
def test__do_node_clean_abort_tear_down_fail(self, tear_mock):
tear_mock.side_effect = Exception('Surprise')
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.CLEANFAIL,
target_provision_state=states.AVAILABLE,
clean_step={'step': 'foo', 'abortable': True})
with task_manager.acquire(self.context, node.uuid) as task:
cleaning.do_node_clean_abort(task)
tear_mock.assert_called_once_with(task.driver.deploy, task)
self.assertIsNotNone(task.node.last_error)
self.assertIsNotNone(task.node.maintenance_reason)
self.assertTrue(task.node.maintenance)
self.assertEqual('clean failure', task.node.fault)