Split cleaning-related functions from manager.py into a new module
With more than 4000 lines of code (more than 8000 of unit tests) the current manager.py is barely manageable. In preparation for the new deployment API, this change moves the internal cleaning-related functions to the new cleaning.py. Change-Id: I13997af2246327bd11b6aaf7029afb20923e64bc Story: #2006910
This commit is contained in:
Dmitry Tantsur
276
ironic/conductor/cleaning.py
Normal file
276
ironic/conductor/cleaning.py
Normal file
@@ -0,0 +1,276 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Functionality related to cleaning."""
|
||||
|
||||
from oslo_log import log
|
||||
|
||||
from ironic.common import exception
|
||||
from ironic.common.i18n import _
|
||||
from ironic.common import states
|
||||
from ironic.conductor import steps as conductor_steps
|
||||
from ironic.conductor import task_manager
|
||||
from ironic.conductor import utils
|
||||
from ironic.conf import CONF
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
@task_manager.require_exclusive_lock
|
||||
def do_node_clean(task, clean_steps=None):
|
||||
"""Internal RPC method to perform cleaning of a node.
|
||||
|
||||
:param task: a TaskManager instance with an exclusive lock on its node
|
||||
:param clean_steps: For a manual clean, the list of clean steps to
|
||||
perform. Is None For automated cleaning (default).
|
||||
For more information, see the clean_steps parameter
|
||||
of :func:`ConductorManager.do_node_clean`.
|
||||
"""
|
||||
node = task.node
|
||||
manual_clean = clean_steps is not None
|
||||
clean_type = 'manual' if manual_clean else 'automated'
|
||||
LOG.debug('Starting %(type)s cleaning for node %(node)s',
|
||||
{'type': clean_type, 'node': node.uuid})
|
||||
|
||||
if not manual_clean and utils.skip_automated_cleaning(node):
|
||||
# Skip cleaning, move to AVAILABLE.
|
||||
node.clean_step = None
|
||||
node.save()
|
||||
|
||||
task.process_event('done')
|
||||
LOG.info('Automated cleaning is disabled, node %s has been '
|
||||
'successfully moved to AVAILABLE state.', node.uuid)
|
||||
return
|
||||
|
||||
# NOTE(dtantsur): this is only reachable during automated cleaning,
|
||||
# for manual cleaning we verify maintenance mode earlier on.
|
||||
if (not CONF.conductor.allow_provisioning_in_maintenance
|
||||
and node.maintenance):
|
||||
msg = _('Cleaning a node in maintenance mode is not allowed')
|
||||
return utils.cleaning_error_handler(task, msg,
|
||||
tear_down_cleaning=False)
|
||||
|
||||
try:
|
||||
# NOTE(ghe): Valid power and network values are needed to perform
|
||||
# a cleaning.
|
||||
task.driver.power.validate(task)
|
||||
task.driver.network.validate(task)
|
||||
except exception.InvalidParameterValue as e:
|
||||
msg = (_('Validation failed. Cannot clean node %(node)s. '
|
||||
'Error: %(msg)s') %
|
||||
{'node': node.uuid, 'msg': e})
|
||||
return utils.cleaning_error_handler(task, msg)
|
||||
|
||||
if manual_clean:
|
||||
info = node.driver_internal_info
|
||||
info['clean_steps'] = clean_steps
|
||||
node.driver_internal_info = info
|
||||
node.save()
|
||||
|
||||
# Do caching of bios settings if supported by driver,
|
||||
# this will be called for both manual and automated cleaning.
|
||||
try:
|
||||
task.driver.bios.cache_bios_settings(task)
|
||||
except exception.UnsupportedDriverExtension:
|
||||
LOG.warning('BIOS settings are not supported for node %s, '
|
||||
'skipping', task.node.uuid)
|
||||
# TODO(zshi) remove this check when classic drivers are removed
|
||||
except Exception:
|
||||
msg = (_('Caching of bios settings failed on node %(node)s. '
|
||||
'Continuing with node cleaning.')
|
||||
% {'node': node.uuid})
|
||||
LOG.exception(msg)
|
||||
|
||||
# Allow the deploy driver to set up the ramdisk again (necessary for
|
||||
# IPA cleaning)
|
||||
try:
|
||||
prepare_result = task.driver.deploy.prepare_cleaning(task)
|
||||
except Exception as e:
|
||||
msg = (_('Failed to prepare node %(node)s for cleaning: %(e)s')
|
||||
% {'node': node.uuid, 'e': e})
|
||||
LOG.exception(msg)
|
||||
return utils.cleaning_error_handler(task, msg)
|
||||
|
||||
if prepare_result == states.CLEANWAIT:
|
||||
# Prepare is asynchronous, the deploy driver will need to
|
||||
# set node.driver_internal_info['clean_steps'] and
|
||||
# node.clean_step and then make an RPC call to
|
||||
# continue_node_clean to start cleaning.
|
||||
|
||||
# For manual cleaning, the target provision state is MANAGEABLE,
|
||||
# whereas for automated cleaning, it is AVAILABLE (the default).
|
||||
target_state = states.MANAGEABLE if manual_clean else None
|
||||
task.process_event('wait', target_state=target_state)
|
||||
return
|
||||
|
||||
try:
|
||||
conductor_steps.set_node_cleaning_steps(task)
|
||||
except (exception.InvalidParameterValue,
|
||||
exception.NodeCleaningFailure) as e:
|
||||
msg = (_('Cannot clean node %(node)s. Error: %(msg)s')
|
||||
% {'node': node.uuid, 'msg': e})
|
||||
return utils.cleaning_error_handler(task, msg)
|
||||
|
||||
steps = node.driver_internal_info.get('clean_steps', [])
|
||||
step_index = 0 if steps else None
|
||||
do_next_clean_step(task, step_index)
|
||||
|
||||
|
||||
@task_manager.require_exclusive_lock
|
||||
def do_next_clean_step(task, step_index):
|
||||
"""Do cleaning, starting from the specified clean step.
|
||||
|
||||
:param task: a TaskManager instance with an exclusive lock
|
||||
:param step_index: The first clean step in the list to execute. This
|
||||
is the index (from 0) into the list of clean steps in the node's
|
||||
driver_internal_info['clean_steps']. Is None if there are no steps
|
||||
to execute.
|
||||
"""
|
||||
node = task.node
|
||||
# For manual cleaning, the target provision state is MANAGEABLE,
|
||||
# whereas for automated cleaning, it is AVAILABLE.
|
||||
manual_clean = node.target_provision_state == states.MANAGEABLE
|
||||
|
||||
if step_index is None:
|
||||
steps = []
|
||||
else:
|
||||
steps = node.driver_internal_info['clean_steps'][step_index:]
|
||||
|
||||
LOG.info('Executing %(state)s on node %(node)s, remaining steps: '
|
||||
'%(steps)s', {'node': node.uuid, 'steps': steps,
|
||||
'state': node.provision_state})
|
||||
|
||||
# Execute each step until we hit an async step or run out of steps
|
||||
for ind, step in enumerate(steps):
|
||||
# Save which step we're about to start so we can restart
|
||||
# if necessary
|
||||
node.clean_step = step
|
||||
driver_internal_info = node.driver_internal_info
|
||||
driver_internal_info['clean_step_index'] = step_index + ind
|
||||
node.driver_internal_info = driver_internal_info
|
||||
node.save()
|
||||
interface = getattr(task.driver, step.get('interface'))
|
||||
LOG.info('Executing %(step)s on node %(node)s',
|
||||
{'step': step, 'node': node.uuid})
|
||||
try:
|
||||
result = interface.execute_clean_step(task, step)
|
||||
except Exception as e:
|
||||
if isinstance(e, exception.AgentConnectionFailed):
|
||||
if task.node.driver_internal_info.get('cleaning_reboot'):
|
||||
LOG.info('Agent is not yet running on node %(node)s '
|
||||
'after cleaning reboot, waiting for agent to '
|
||||
'come up to run next clean step %(step)s.',
|
||||
{'node': node.uuid, 'step': step})
|
||||
driver_internal_info['skip_current_clean_step'] = False
|
||||
node.driver_internal_info = driver_internal_info
|
||||
target_state = (states.MANAGEABLE if manual_clean
|
||||
else None)
|
||||
task.process_event('wait', target_state=target_state)
|
||||
return
|
||||
|
||||
msg = (_('Node %(node)s failed step %(step)s: '
|
||||
'%(exc)s') %
|
||||
{'node': node.uuid, 'exc': e,
|
||||
'step': node.clean_step})
|
||||
LOG.exception(msg)
|
||||
utils.cleaning_error_handler(task, msg)
|
||||
return
|
||||
|
||||
# Check if the step is done or not. The step should return
|
||||
# states.CLEANWAIT if the step is still being executed, or
|
||||
# None if the step is done.
|
||||
if result == states.CLEANWAIT:
|
||||
# Kill this worker, the async step will make an RPC call to
|
||||
# continue_node_clean to continue cleaning
|
||||
LOG.info('Clean step %(step)s on node %(node)s being '
|
||||
'executed asynchronously, waiting for driver.',
|
||||
{'node': node.uuid, 'step': step})
|
||||
target_state = states.MANAGEABLE if manual_clean else None
|
||||
task.process_event('wait', target_state=target_state)
|
||||
return
|
||||
elif result is not None:
|
||||
msg = (_('While executing step %(step)s on node '
|
||||
'%(node)s, step returned invalid value: %(val)s')
|
||||
% {'step': step, 'node': node.uuid, 'val': result})
|
||||
LOG.error(msg)
|
||||
return utils.cleaning_error_handler(task, msg)
|
||||
LOG.info('Node %(node)s finished clean step %(step)s',
|
||||
{'node': node.uuid, 'step': step})
|
||||
|
||||
# Clear clean_step
|
||||
node.clean_step = None
|
||||
driver_internal_info = node.driver_internal_info
|
||||
driver_internal_info['clean_steps'] = None
|
||||
driver_internal_info.pop('clean_step_index', None)
|
||||
driver_internal_info.pop('cleaning_reboot', None)
|
||||
driver_internal_info.pop('cleaning_polling', None)
|
||||
# Remove agent_url
|
||||
if not utils.fast_track_able(task):
|
||||
driver_internal_info.pop('agent_url', None)
|
||||
node.driver_internal_info = driver_internal_info
|
||||
node.save()
|
||||
try:
|
||||
task.driver.deploy.tear_down_cleaning(task)
|
||||
except Exception as e:
|
||||
msg = (_('Failed to tear down from cleaning for node %(node)s, '
|
||||
'reason: %(err)s')
|
||||
% {'node': node.uuid, 'err': e})
|
||||
LOG.exception(msg)
|
||||
return utils.cleaning_error_handler(task, msg,
|
||||
tear_down_cleaning=False)
|
||||
|
||||
LOG.info('Node %s cleaning complete', node.uuid)
|
||||
event = 'manage' if manual_clean or node.retired else 'done'
|
||||
# NOTE(rloo): No need to specify target prov. state; we're done
|
||||
task.process_event(event)
|
||||
|
||||
|
||||
@task_manager.require_exclusive_lock
|
||||
def do_node_clean_abort(task, step_name=None):
|
||||
"""Internal method to abort an ongoing operation.
|
||||
|
||||
:param task: a TaskManager instance with an exclusive lock
|
||||
:param step_name: The name of the clean step.
|
||||
"""
|
||||
node = task.node
|
||||
try:
|
||||
task.driver.deploy.tear_down_cleaning(task)
|
||||
except Exception as e:
|
||||
LOG.exception('Failed to tear down cleaning for node %(node)s '
|
||||
'after aborting the operation. Error: %(err)s',
|
||||
{'node': node.uuid, 'err': e})
|
||||
error_msg = _('Failed to tear down cleaning after aborting '
|
||||
'the operation')
|
||||
utils.cleaning_error_handler(task, error_msg,
|
||||
tear_down_cleaning=False,
|
||||
set_fail_state=False)
|
||||
return
|
||||
|
||||
info_message = _('Clean operation aborted for node %s') % node.uuid
|
||||
last_error = _('By request, the clean operation was aborted')
|
||||
if step_name:
|
||||
msg = _(' after the completion of step "%s"') % step_name
|
||||
last_error += msg
|
||||
info_message += msg
|
||||
|
||||
node.last_error = last_error
|
||||
node.clean_step = None
|
||||
info = node.driver_internal_info
|
||||
# Clear any leftover metadata about cleaning
|
||||
info.pop('clean_step_index', None)
|
||||
info.pop('cleaning_reboot', None)
|
||||
info.pop('cleaning_polling', None)
|
||||
info.pop('skip_current_clean_step', None)
|
||||
info.pop('agent_url', None)
|
||||
node.driver_internal_info = info
|
||||
node.save()
|
||||
LOG.info(info_message)
|
||||
@@ -64,6 +64,7 @@ from ironic.common import nova
|
||||
from ironic.common import states
|
||||
from ironic.conductor import allocations
|
||||
from ironic.conductor import base_manager
|
||||
from ironic.conductor import cleaning
|
||||
from ironic.conductor import deployments
|
||||
from ironic.conductor import notification_utils as notify_utils
|
||||
from ironic.conductor import steps as conductor_steps
|
||||
@@ -1086,7 +1087,7 @@ class ConductorManager(base_manager.BaseConductorManager):
|
||||
|
||||
# Begin cleaning
|
||||
task.process_event('clean')
|
||||
self._do_node_clean(task)
|
||||
cleaning.do_node_clean(task)
|
||||
|
||||
@METRICS.timer('ConductorManager.do_node_clean')
|
||||
@messaging.expected_exceptions(exception.InvalidParameterValue,
|
||||
@@ -1133,10 +1134,10 @@ class ConductorManager(base_manager.BaseConductorManager):
|
||||
raise exception.NodeInMaintenance(op=_('cleaning'),
|
||||
node=node.uuid)
|
||||
|
||||
# NOTE(rloo): _do_node_clean() will also make similar calls to
|
||||
# validate power & network, but we are doing it again here so that
|
||||
# the user gets immediate feedback of any issues. This behaviour
|
||||
# (of validating) is consistent with other methods like
|
||||
# NOTE(rloo): cleaning.do_node_clean() will also make similar calls
|
||||
# to validate power & network, but we are doing it again here so
|
||||
# that the user gets immediate feedback of any issues. This
|
||||
# behaviour (of validating) is consistent with other methods like
|
||||
# self.do_node_deploy().
|
||||
try:
|
||||
task.driver.power.validate(task)
|
||||
@@ -1151,7 +1152,7 @@ class ConductorManager(base_manager.BaseConductorManager):
|
||||
task.process_event(
|
||||
'clean',
|
||||
callback=self._spawn_worker,
|
||||
call_args=(self._do_node_clean, task, clean_steps),
|
||||
call_args=(cleaning.do_node_clean, task, clean_steps),
|
||||
err_handler=utils.provisioning_error_handler,
|
||||
target_state=states.MANAGEABLE)
|
||||
except exception.InvalidState:
|
||||
@@ -1224,7 +1225,7 @@ class ConductorManager(base_manager.BaseConductorManager):
|
||||
task.process_event(
|
||||
'abort',
|
||||
callback=self._spawn_worker,
|
||||
call_args=(self._do_node_clean_abort,
|
||||
call_args=(cleaning.do_node_clean_abort,
|
||||
task, step_name),
|
||||
err_handler=utils.provisioning_error_handler,
|
||||
target_state=target_state)
|
||||
@@ -1243,216 +1244,9 @@ class ConductorManager(base_manager.BaseConductorManager):
|
||||
task.node)
|
||||
task.spawn_after(
|
||||
self._spawn_worker,
|
||||
self._do_next_clean_step,
|
||||
cleaning.do_next_clean_step,
|
||||
task, next_step_index)
|
||||
|
||||
@task_manager.require_exclusive_lock
|
||||
def _do_node_clean(self, task, clean_steps=None):
|
||||
"""Internal RPC method to perform cleaning of a node.
|
||||
|
||||
:param task: a TaskManager instance with an exclusive lock on its node
|
||||
:param clean_steps: For a manual clean, the list of clean steps to
|
||||
perform. Is None For automated cleaning (default).
|
||||
For more information, see the clean_steps parameter
|
||||
of :func:`ConductorManager.do_node_clean`.
|
||||
"""
|
||||
node = task.node
|
||||
manual_clean = clean_steps is not None
|
||||
clean_type = 'manual' if manual_clean else 'automated'
|
||||
LOG.debug('Starting %(type)s cleaning for node %(node)s',
|
||||
{'type': clean_type, 'node': node.uuid})
|
||||
|
||||
if not manual_clean and utils.skip_automated_cleaning(node):
|
||||
# Skip cleaning, move to AVAILABLE.
|
||||
node.clean_step = None
|
||||
node.save()
|
||||
|
||||
task.process_event('done')
|
||||
LOG.info('Automated cleaning is disabled, node %s has been '
|
||||
'successfully moved to AVAILABLE state.', node.uuid)
|
||||
return
|
||||
|
||||
# NOTE(dtantsur): this is only reachable during automated cleaning,
|
||||
# for manual cleaning we verify maintenance mode earlier on.
|
||||
if (not CONF.conductor.allow_provisioning_in_maintenance
|
||||
and node.maintenance):
|
||||
msg = _('Cleaning a node in maintenance mode is not allowed')
|
||||
return utils.cleaning_error_handler(task, msg,
|
||||
tear_down_cleaning=False)
|
||||
|
||||
try:
|
||||
# NOTE(ghe): Valid power and network values are needed to perform
|
||||
# a cleaning.
|
||||
task.driver.power.validate(task)
|
||||
task.driver.network.validate(task)
|
||||
except exception.InvalidParameterValue as e:
|
||||
msg = (_('Validation failed. Cannot clean node %(node)s. '
|
||||
'Error: %(msg)s') %
|
||||
{'node': node.uuid, 'msg': e})
|
||||
return utils.cleaning_error_handler(task, msg)
|
||||
|
||||
if manual_clean:
|
||||
info = node.driver_internal_info
|
||||
info['clean_steps'] = clean_steps
|
||||
node.driver_internal_info = info
|
||||
node.save()
|
||||
|
||||
# Do caching of bios settings if supported by driver,
|
||||
# this will be called for both manual and automated cleaning.
|
||||
try:
|
||||
task.driver.bios.cache_bios_settings(task)
|
||||
except exception.UnsupportedDriverExtension:
|
||||
LOG.warning('BIOS settings are not supported for node %s, '
|
||||
'skipping', task.node.uuid)
|
||||
# TODO(zshi) remove this check when classic drivers are removed
|
||||
except Exception:
|
||||
msg = (_('Caching of bios settings failed on node %(node)s. '
|
||||
'Continuing with node cleaning.')
|
||||
% {'node': node.uuid})
|
||||
LOG.exception(msg)
|
||||
|
||||
# Allow the deploy driver to set up the ramdisk again (necessary for
|
||||
# IPA cleaning)
|
||||
try:
|
||||
prepare_result = task.driver.deploy.prepare_cleaning(task)
|
||||
except Exception as e:
|
||||
msg = (_('Failed to prepare node %(node)s for cleaning: %(e)s')
|
||||
% {'node': node.uuid, 'e': e})
|
||||
LOG.exception(msg)
|
||||
return utils.cleaning_error_handler(task, msg)
|
||||
|
||||
if prepare_result == states.CLEANWAIT:
|
||||
# Prepare is asynchronous, the deploy driver will need to
|
||||
# set node.driver_internal_info['clean_steps'] and
|
||||
# node.clean_step and then make an RPC call to
|
||||
# continue_node_clean to start cleaning.
|
||||
|
||||
# For manual cleaning, the target provision state is MANAGEABLE,
|
||||
# whereas for automated cleaning, it is AVAILABLE (the default).
|
||||
target_state = states.MANAGEABLE if manual_clean else None
|
||||
task.process_event('wait', target_state=target_state)
|
||||
return
|
||||
|
||||
try:
|
||||
conductor_steps.set_node_cleaning_steps(task)
|
||||
except (exception.InvalidParameterValue,
|
||||
exception.NodeCleaningFailure) as e:
|
||||
msg = (_('Cannot clean node %(node)s. Error: %(msg)s')
|
||||
% {'node': node.uuid, 'msg': e})
|
||||
return utils.cleaning_error_handler(task, msg)
|
||||
|
||||
steps = node.driver_internal_info.get('clean_steps', [])
|
||||
step_index = 0 if steps else None
|
||||
self._do_next_clean_step(task, step_index)
|
||||
|
||||
@task_manager.require_exclusive_lock
|
||||
def _do_next_clean_step(self, task, step_index):
|
||||
"""Do cleaning, starting from the specified clean step.
|
||||
|
||||
:param task: a TaskManager instance with an exclusive lock
|
||||
:param step_index: The first clean step in the list to execute. This
|
||||
is the index (from 0) into the list of clean steps in the node's
|
||||
driver_internal_info['clean_steps']. Is None if there are no steps
|
||||
to execute.
|
||||
"""
|
||||
node = task.node
|
||||
# For manual cleaning, the target provision state is MANAGEABLE,
|
||||
# whereas for automated cleaning, it is AVAILABLE.
|
||||
manual_clean = node.target_provision_state == states.MANAGEABLE
|
||||
|
||||
if step_index is None:
|
||||
steps = []
|
||||
else:
|
||||
steps = node.driver_internal_info['clean_steps'][step_index:]
|
||||
|
||||
LOG.info('Executing %(state)s on node %(node)s, remaining steps: '
|
||||
'%(steps)s', {'node': node.uuid, 'steps': steps,
|
||||
'state': node.provision_state})
|
||||
|
||||
# Execute each step until we hit an async step or run out of steps
|
||||
for ind, step in enumerate(steps):
|
||||
# Save which step we're about to start so we can restart
|
||||
# if necessary
|
||||
node.clean_step = step
|
||||
driver_internal_info = node.driver_internal_info
|
||||
driver_internal_info['clean_step_index'] = step_index + ind
|
||||
node.driver_internal_info = driver_internal_info
|
||||
node.save()
|
||||
interface = getattr(task.driver, step.get('interface'))
|
||||
LOG.info('Executing %(step)s on node %(node)s',
|
||||
{'step': step, 'node': node.uuid})
|
||||
try:
|
||||
result = interface.execute_clean_step(task, step)
|
||||
except Exception as e:
|
||||
if isinstance(e, exception.AgentConnectionFailed):
|
||||
if task.node.driver_internal_info.get('cleaning_reboot'):
|
||||
LOG.info('Agent is not yet running on node %(node)s '
|
||||
'after cleaning reboot, waiting for agent to '
|
||||
'come up to run next clean step %(step)s.',
|
||||
{'node': node.uuid, 'step': step})
|
||||
driver_internal_info['skip_current_clean_step'] = False
|
||||
node.driver_internal_info = driver_internal_info
|
||||
target_state = (states.MANAGEABLE if manual_clean
|
||||
else None)
|
||||
task.process_event('wait', target_state=target_state)
|
||||
return
|
||||
|
||||
msg = (_('Node %(node)s failed step %(step)s: '
|
||||
'%(exc)s') %
|
||||
{'node': node.uuid, 'exc': e,
|
||||
'step': node.clean_step})
|
||||
LOG.exception(msg)
|
||||
utils.cleaning_error_handler(task, msg)
|
||||
return
|
||||
|
||||
# Check if the step is done or not. The step should return
|
||||
# states.CLEANWAIT if the step is still being executed, or
|
||||
# None if the step is done.
|
||||
if result == states.CLEANWAIT:
|
||||
# Kill this worker, the async step will make an RPC call to
|
||||
# continue_node_clean to continue cleaning
|
||||
LOG.info('Clean step %(step)s on node %(node)s being '
|
||||
'executed asynchronously, waiting for driver.',
|
||||
{'node': node.uuid, 'step': step})
|
||||
target_state = states.MANAGEABLE if manual_clean else None
|
||||
task.process_event('wait', target_state=target_state)
|
||||
return
|
||||
elif result is not None:
|
||||
msg = (_('While executing step %(step)s on node '
|
||||
'%(node)s, step returned invalid value: %(val)s')
|
||||
% {'step': step, 'node': node.uuid, 'val': result})
|
||||
LOG.error(msg)
|
||||
return utils.cleaning_error_handler(task, msg)
|
||||
LOG.info('Node %(node)s finished clean step %(step)s',
|
||||
{'node': node.uuid, 'step': step})
|
||||
|
||||
# Clear clean_step
|
||||
node.clean_step = None
|
||||
driver_internal_info = node.driver_internal_info
|
||||
driver_internal_info['clean_steps'] = None
|
||||
driver_internal_info.pop('clean_step_index', None)
|
||||
driver_internal_info.pop('cleaning_reboot', None)
|
||||
driver_internal_info.pop('cleaning_polling', None)
|
||||
# Remove agent_url
|
||||
if not utils.fast_track_able(task):
|
||||
driver_internal_info.pop('agent_url', None)
|
||||
node.driver_internal_info = driver_internal_info
|
||||
node.save()
|
||||
try:
|
||||
task.driver.deploy.tear_down_cleaning(task)
|
||||
except Exception as e:
|
||||
msg = (_('Failed to tear down from cleaning for node %(node)s, '
|
||||
'reason: %(err)s')
|
||||
% {'node': node.uuid, 'err': e})
|
||||
LOG.exception(msg)
|
||||
return utils.cleaning_error_handler(task, msg,
|
||||
tear_down_cleaning=False)
|
||||
|
||||
LOG.info('Node %s cleaning complete', node.uuid)
|
||||
event = 'manage' if manual_clean or node.retired else 'done'
|
||||
# NOTE(rloo): No need to specify target prov. state; we're done
|
||||
task.process_event(event)
|
||||
|
||||
@task_manager.require_exclusive_lock
|
||||
def _do_node_verify(self, task):
|
||||
"""Internal method to perform power credentials verification."""
|
||||
@@ -1489,47 +1283,6 @@ class ConductorManager(base_manager.BaseConductorManager):
|
||||
node.last_error = error
|
||||
task.process_event('fail')
|
||||
|
||||
@task_manager.require_exclusive_lock
|
||||
def _do_node_clean_abort(self, task, step_name=None):
|
||||
"""Internal method to abort an ongoing operation.
|
||||
|
||||
:param task: a TaskManager instance with an exclusive lock
|
||||
:param step_name: The name of the clean step.
|
||||
"""
|
||||
node = task.node
|
||||
try:
|
||||
task.driver.deploy.tear_down_cleaning(task)
|
||||
except Exception as e:
|
||||
LOG.exception('Failed to tear down cleaning for node %(node)s '
|
||||
'after aborting the operation. Error: %(err)s',
|
||||
{'node': node.uuid, 'err': e})
|
||||
error_msg = _('Failed to tear down cleaning after aborting '
|
||||
'the operation')
|
||||
utils.cleaning_error_handler(task, error_msg,
|
||||
tear_down_cleaning=False,
|
||||
set_fail_state=False)
|
||||
return
|
||||
|
||||
info_message = _('Clean operation aborted for node %s') % node.uuid
|
||||
last_error = _('By request, the clean operation was aborted')
|
||||
if step_name:
|
||||
msg = _(' after the completion of step "%s"') % step_name
|
||||
last_error += msg
|
||||
info_message += msg
|
||||
|
||||
node.last_error = last_error
|
||||
node.clean_step = None
|
||||
info = node.driver_internal_info
|
||||
# Clear any leftover metadata about cleaning
|
||||
info.pop('clean_step_index', None)
|
||||
info.pop('cleaning_reboot', None)
|
||||
info.pop('cleaning_polling', None)
|
||||
info.pop('skip_current_clean_step', None)
|
||||
info.pop('agent_url', None)
|
||||
node.driver_internal_info = info
|
||||
node.save()
|
||||
LOG.info(info_message)
|
||||
|
||||
@METRICS.timer('ConductorManager.do_provisioning_action')
|
||||
@messaging.expected_exceptions(exception.NoFreeConductorWorker,
|
||||
exception.NodeLocked,
|
||||
@@ -1568,7 +1321,7 @@ class ConductorManager(base_manager.BaseConductorManager):
|
||||
task.process_event(
|
||||
'provide',
|
||||
callback=self._spawn_worker,
|
||||
call_args=(self._do_node_clean, task),
|
||||
call_args=(cleaning.do_node_clean, task),
|
||||
err_handler=utils.provisioning_error_handler)
|
||||
return
|
||||
|
||||
@@ -1640,7 +1393,7 @@ class ConductorManager(base_manager.BaseConductorManager):
|
||||
task.process_event(
|
||||
'abort',
|
||||
callback=self._spawn_worker,
|
||||
call_args=(self._do_node_clean_abort, task),
|
||||
call_args=(cleaning.do_node_clean_abort, task),
|
||||
err_handler=utils.provisioning_error_handler,
|
||||
target_state=target_state)
|
||||
return
|
||||
|
||||
975
ironic/tests/unit/conductor/test_cleaning.py
Normal file
975
ironic/tests/unit/conductor/test_cleaning.py
Normal file
@@ -0,0 +1,975 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Tests for cleaning bits."""
|
||||
|
||||
import mock
|
||||
from oslo_config import cfg
|
||||
from oslo_utils import uuidutils
|
||||
|
||||
from ironic.common import exception
|
||||
from ironic.common import states
|
||||
from ironic.conductor import cleaning
|
||||
from ironic.conductor import steps as conductor_steps
|
||||
from ironic.conductor import task_manager
|
||||
from ironic.drivers.modules import fake
|
||||
from ironic.drivers.modules.network import flat as n_flat
|
||||
from ironic.tests.unit.db import base as db_base
|
||||
from ironic.tests.unit.objects import utils as obj_utils
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class DoNodeCleanTestCase(db_base.DbTestCase):
|
||||
def setUp(self):
|
||||
super(DoNodeCleanTestCase, self).setUp()
|
||||
self.config(automated_clean=True, group='conductor')
|
||||
self.power_update = {
|
||||
'step': 'update_firmware', 'priority': 10, 'interface': 'power'}
|
||||
self.deploy_update = {
|
||||
'step': 'update_firmware', 'priority': 10, 'interface': 'deploy'}
|
||||
self.deploy_erase = {
|
||||
'step': 'erase_disks', 'priority': 20, 'interface': 'deploy'}
|
||||
# Automated cleaning should be executed in this order
|
||||
self.clean_steps = [self.deploy_erase, self.power_update,
|
||||
self.deploy_update]
|
||||
self.next_clean_step_index = 1
|
||||
# Manual clean step
|
||||
self.deploy_raid = {
|
||||
'step': 'build_raid', 'priority': 0, 'interface': 'deploy'}
|
||||
|
||||
def __do_node_clean_validate_fail(self, mock_validate, clean_steps=None):
|
||||
# InvalidParameterValue should cause node to go to CLEANFAIL
|
||||
mock_validate.side_effect = exception.InvalidParameterValue('error')
|
||||
tgt_prov_state = states.MANAGEABLE if clean_steps else states.AVAILABLE
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state)
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task, clean_steps=clean_steps)
|
||||
node.refresh()
|
||||
self.assertEqual(states.CLEANFAIL, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
||||
autospec=True)
|
||||
def test__do_node_clean_automated_power_validate_fail(self, mock_validate):
|
||||
self.__do_node_clean_validate_fail(mock_validate)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
||||
autospec=True)
|
||||
def test__do_node_clean_manual_power_validate_fail(self, mock_validate):
|
||||
self.__do_node_clean_validate_fail(mock_validate, clean_steps=[])
|
||||
|
||||
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
||||
autospec=True)
|
||||
def test__do_node_clean_automated_network_validate_fail(self,
|
||||
mock_validate):
|
||||
self.__do_node_clean_validate_fail(mock_validate)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
||||
autospec=True)
|
||||
def test__do_node_clean_manual_network_validate_fail(self, mock_validate):
|
||||
self.__do_node_clean_validate_fail(mock_validate, clean_steps=[])
|
||||
|
||||
@mock.patch.object(cleaning, 'LOG', autospec=True)
|
||||
@mock.patch.object(conductor_steps, 'set_node_cleaning_steps',
|
||||
autospec=True)
|
||||
@mock.patch.object(cleaning, 'do_next_clean_step', autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_cleaning',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeBIOS.cache_bios_settings',
|
||||
autospec=True)
|
||||
def _test__do_node_clean_cache_bios(self, mock_bios, mock_validate,
|
||||
mock_prep, mock_next_step, mock_steps,
|
||||
mock_log, clean_steps=None,
|
||||
enable_unsupported=False,
|
||||
enable_exception=False):
|
||||
if enable_unsupported:
|
||||
mock_bios.side_effect = exception.UnsupportedDriverExtension('')
|
||||
elif enable_exception:
|
||||
mock_bios.side_effect = exception.IronicException('test')
|
||||
mock_prep.return_value = states.NOSTATE
|
||||
tgt_prov_state = states.MANAGEABLE if clean_steps else states.AVAILABLE
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state)
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task, clean_steps=clean_steps)
|
||||
node.refresh()
|
||||
mock_bios.assert_called_once_with(mock.ANY, task)
|
||||
if clean_steps:
|
||||
self.assertEqual(states.CLEANING, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
else:
|
||||
self.assertEqual(states.CLEANING, node.provision_state)
|
||||
self.assertEqual(states.AVAILABLE, node.target_provision_state)
|
||||
mock_validate.assert_called_once_with(mock.ANY, task)
|
||||
if enable_exception:
|
||||
mock_log.exception.assert_called_once_with(
|
||||
'Caching of bios settings failed on node {}. '
|
||||
'Continuing with node cleaning.'
|
||||
.format(node.uuid))
|
||||
|
||||
def test__do_node_clean_manual_cache_bios(self):
|
||||
self._test__do_node_clean_cache_bios(clean_steps=[self.deploy_raid])
|
||||
|
||||
def test__do_node_clean_automated_cache_bios(self):
|
||||
self._test__do_node_clean_cache_bios()
|
||||
|
||||
def test__do_node_clean_manual_cache_bios_exception(self):
|
||||
self._test__do_node_clean_cache_bios(clean_steps=[self.deploy_raid],
|
||||
enable_exception=True)
|
||||
|
||||
def test__do_node_clean_automated_cache_bios_exception(self):
|
||||
self._test__do_node_clean_cache_bios(enable_exception=True)
|
||||
|
||||
def test__do_node_clean_manual_cache_bios_unsupported(self):
|
||||
self._test__do_node_clean_cache_bios(clean_steps=[self.deploy_raid],
|
||||
enable_unsupported=True)
|
||||
|
||||
def test__do_node_clean_automated_cache_bios_unsupported(self):
|
||||
self._test__do_node_clean_cache_bios(enable_unsupported=True)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
||||
autospec=True)
|
||||
def test__do_node_clean_automated_disabled(self, mock_validate):
|
||||
self.config(automated_clean=False, group='conductor')
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=states.AVAILABLE,
|
||||
last_error=None)
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task)
|
||||
node.refresh()
|
||||
|
||||
# Assert that the node was moved to available without cleaning
|
||||
self.assertFalse(mock_validate.called)
|
||||
self.assertEqual(states.AVAILABLE, node.provision_state)
|
||||
self.assertEqual(states.NOSTATE, node.target_provision_state)
|
||||
self.assertEqual({}, node.clean_step)
|
||||
self.assertNotIn('clean_steps', node.driver_internal_info)
|
||||
self.assertNotIn('clean_step_index', node.driver_internal_info)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
||||
autospec=True)
|
||||
def test__do_node_clean_automated_disabled_individual_enabled(
|
||||
self, mock_network, mock_validate):
|
||||
self.config(automated_clean=False, group='conductor')
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=states.AVAILABLE,
|
||||
last_error=None, automated_clean=True)
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task)
|
||||
node.refresh()
|
||||
|
||||
# Assert that the node clean was called
|
||||
self.assertTrue(mock_validate.called)
|
||||
self.assertIn('clean_steps', node.driver_internal_info)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
||||
autospec=True)
|
||||
def test__do_node_clean_automated_disabled_individual_disabled(
|
||||
self, mock_validate):
|
||||
self.config(automated_clean=False, group='conductor')
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=states.AVAILABLE,
|
||||
last_error=None, automated_clean=False)
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task)
|
||||
node.refresh()
|
||||
|
||||
# Assert that the node was moved to available without cleaning
|
||||
self.assertFalse(mock_validate.called)
|
||||
self.assertEqual(states.AVAILABLE, node.provision_state)
|
||||
self.assertEqual(states.NOSTATE, node.target_provision_state)
|
||||
self.assertEqual({}, node.clean_step)
|
||||
self.assertNotIn('clean_steps', node.driver_internal_info)
|
||||
self.assertNotIn('clean_step_index', node.driver_internal_info)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
||||
autospec=True)
|
||||
def test__do_node_clean_automated_enabled(self, mock_validate,
|
||||
mock_network):
|
||||
self.config(automated_clean=True, group='conductor')
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=states.AVAILABLE,
|
||||
last_error=None,
|
||||
driver_internal_info={'agent_url': 'url'})
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task)
|
||||
node.refresh()
|
||||
|
||||
# Assert that the node was cleaned
|
||||
self.assertTrue(mock_validate.called)
|
||||
self.assertIn('clean_steps', node.driver_internal_info)
|
||||
self.assertNotIn('agent_url', node.driver_internal_info)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
||||
autospec=True)
|
||||
def test__do_node_clean_automated_enabled_individual_enabled(
|
||||
self, mock_network, mock_validate):
|
||||
self.config(automated_clean=True, group='conductor')
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=states.AVAILABLE,
|
||||
last_error=None, automated_clean=True)
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task)
|
||||
node.refresh()
|
||||
|
||||
# Assert that the node was cleaned
|
||||
self.assertTrue(mock_validate.called)
|
||||
self.assertIn('clean_steps', node.driver_internal_info)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
||||
autospec=True)
|
||||
def test__do_node_clean_automated_enabled_individual_none(
|
||||
self, mock_validate, mock_network):
|
||||
self.config(automated_clean=True, group='conductor')
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=states.AVAILABLE,
|
||||
last_error=None, automated_clean=None)
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task)
|
||||
node.refresh()
|
||||
|
||||
# Assert that the node was cleaned
|
||||
self.assertTrue(mock_validate.called)
|
||||
self.assertIn('clean_steps', node.driver_internal_info)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.tear_down_cleaning',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_cleaning',
|
||||
autospec=True)
|
||||
def test__do_node_clean_maintenance(self, mock_prep, mock_tear_down):
|
||||
CONF.set_override('allow_provisioning_in_maintenance', False,
|
||||
group='conductor')
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=states.AVAILABLE,
|
||||
maintenance=True,
|
||||
maintenance_reason='Original reason')
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task)
|
||||
node.refresh()
|
||||
self.assertEqual(states.CLEANFAIL, node.provision_state)
|
||||
self.assertEqual(states.AVAILABLE, node.target_provision_state)
|
||||
self.assertIn('is not allowed', node.last_error)
|
||||
self.assertTrue(node.maintenance)
|
||||
self.assertEqual('Original reason', node.maintenance_reason)
|
||||
self.assertFalse(mock_prep.called)
|
||||
self.assertFalse(mock_tear_down.called)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_cleaning',
|
||||
autospec=True)
|
||||
def __do_node_clean_prepare_clean_fail(self, mock_prep, mock_validate,
|
||||
clean_steps=None):
|
||||
# Exception from task.driver.deploy.prepare_cleaning should cause node
|
||||
# to go to CLEANFAIL
|
||||
mock_prep.side_effect = exception.InvalidParameterValue('error')
|
||||
tgt_prov_state = states.MANAGEABLE if clean_steps else states.AVAILABLE
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state)
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task, clean_steps=clean_steps)
|
||||
node.refresh()
|
||||
self.assertEqual(states.CLEANFAIL, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
mock_prep.assert_called_once_with(mock.ANY, task)
|
||||
mock_validate.assert_called_once_with(mock.ANY, task)
|
||||
|
||||
def test__do_node_clean_automated_prepare_clean_fail(self):
|
||||
self.__do_node_clean_prepare_clean_fail()
|
||||
|
||||
def test__do_node_clean_manual_prepare_clean_fail(self):
|
||||
self.__do_node_clean_prepare_clean_fail(clean_steps=[self.deploy_raid])
|
||||
|
||||
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare_cleaning',
|
||||
autospec=True)
|
||||
def __do_node_clean_prepare_clean_wait(self, mock_prep, mock_validate,
|
||||
clean_steps=None):
|
||||
mock_prep.return_value = states.CLEANWAIT
|
||||
tgt_prov_state = states.MANAGEABLE if clean_steps else states.AVAILABLE
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state)
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task, clean_steps=clean_steps)
|
||||
node.refresh()
|
||||
self.assertEqual(states.CLEANWAIT, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
mock_prep.assert_called_once_with(mock.ANY, mock.ANY)
|
||||
mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
|
||||
|
||||
def test__do_node_clean_automated_prepare_clean_wait(self):
|
||||
self.__do_node_clean_prepare_clean_wait()
|
||||
|
||||
def test__do_node_clean_manual_prepare_clean_wait(self):
|
||||
self.__do_node_clean_prepare_clean_wait(clean_steps=[self.deploy_raid])
|
||||
|
||||
@mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
|
||||
@mock.patch.object(conductor_steps, 'set_node_cleaning_steps',
|
||||
autospec=True)
|
||||
def __do_node_clean_steps_fail(self, mock_steps, mock_validate,
|
||||
clean_steps=None, invalid_exc=True):
|
||||
if invalid_exc:
|
||||
mock_steps.side_effect = exception.InvalidParameterValue('invalid')
|
||||
else:
|
||||
mock_steps.side_effect = exception.NodeCleaningFailure('failure')
|
||||
tgt_prov_state = states.MANAGEABLE if clean_steps else states.AVAILABLE
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
uuid=uuidutils.generate_uuid(),
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state)
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task, clean_steps=clean_steps)
|
||||
mock_validate.assert_called_once_with(mock.ANY, task)
|
||||
node.refresh()
|
||||
self.assertEqual(states.CLEANFAIL, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
mock_steps.assert_called_once_with(mock.ANY)
|
||||
|
||||
def test__do_node_clean_automated_steps_fail(self):
|
||||
for invalid in (True, False):
|
||||
self.__do_node_clean_steps_fail(invalid_exc=invalid)
|
||||
|
||||
def test__do_node_clean_manual_steps_fail(self):
|
||||
for invalid in (True, False):
|
||||
self.__do_node_clean_steps_fail(clean_steps=[self.deploy_raid],
|
||||
invalid_exc=invalid)
|
||||
|
||||
@mock.patch.object(conductor_steps, 'set_node_cleaning_steps',
|
||||
autospec=True)
|
||||
@mock.patch.object(cleaning, 'do_next_clean_step', autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.validate',
|
||||
autospec=True)
|
||||
def __do_node_clean(self, mock_power_valid, mock_network_valid,
|
||||
mock_next_step, mock_steps, clean_steps=None):
|
||||
if clean_steps:
|
||||
tgt_prov_state = states.MANAGEABLE
|
||||
driver_info = {}
|
||||
else:
|
||||
tgt_prov_state = states.AVAILABLE
|
||||
driver_info = {'clean_steps': self.clean_steps}
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state,
|
||||
last_error=None,
|
||||
power_state=states.POWER_OFF,
|
||||
driver_internal_info=driver_info)
|
||||
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_node_clean(task, clean_steps=clean_steps)
|
||||
|
||||
node.refresh()
|
||||
|
||||
mock_power_valid.assert_called_once_with(mock.ANY, task)
|
||||
mock_network_valid.assert_called_once_with(mock.ANY, task)
|
||||
mock_next_step.assert_called_once_with(task, 0)
|
||||
mock_steps.assert_called_once_with(task)
|
||||
if clean_steps:
|
||||
self.assertEqual(clean_steps,
|
||||
node.driver_internal_info['clean_steps'])
|
||||
|
||||
# Check that state didn't change
|
||||
self.assertEqual(states.CLEANING, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
|
||||
def test__do_node_clean_automated(self):
|
||||
self.__do_node_clean()
|
||||
|
||||
def test__do_node_clean_manual(self):
|
||||
self.__do_node_clean(clean_steps=[self.deploy_raid])
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
|
||||
autospec=True)
|
||||
def _do_next_clean_step_first_step_async(self, return_state, mock_execute,
|
||||
clean_steps=None):
|
||||
# Execute the first async clean step on a node
|
||||
driver_internal_info = {'clean_step_index': None}
|
||||
if clean_steps:
|
||||
tgt_prov_state = states.MANAGEABLE
|
||||
driver_internal_info['clean_steps'] = clean_steps
|
||||
else:
|
||||
tgt_prov_state = states.AVAILABLE
|
||||
driver_internal_info['clean_steps'] = self.clean_steps
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state,
|
||||
last_error=None,
|
||||
driver_internal_info=driver_internal_info,
|
||||
clean_step={})
|
||||
mock_execute.return_value = return_state
|
||||
expected_first_step = node.driver_internal_info['clean_steps'][0]
|
||||
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_next_clean_step(task, 0)
|
||||
|
||||
node.refresh()
|
||||
|
||||
self.assertEqual(states.CLEANWAIT, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
self.assertEqual(expected_first_step, node.clean_step)
|
||||
self.assertEqual(0, node.driver_internal_info['clean_step_index'])
|
||||
mock_execute.assert_called_once_with(
|
||||
mock.ANY, mock.ANY, expected_first_step)
|
||||
|
||||
def test_do_next_clean_step_automated_first_step_async(self):
|
||||
self._do_next_clean_step_first_step_async(states.CLEANWAIT)
|
||||
|
||||
def test_do_next_clean_step_manual_first_step_async(self):
|
||||
self._do_next_clean_step_first_step_async(
|
||||
states.CLEANWAIT, clean_steps=[self.deploy_raid])
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_clean_step',
|
||||
autospec=True)
|
||||
def _do_next_clean_step_continue_from_last_cleaning(self, return_state,
|
||||
mock_execute,
|
||||
manual=False):
|
||||
# Resume an in-progress cleaning after the first async step
|
||||
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state,
|
||||
last_error=None,
|
||||
driver_internal_info={'clean_steps': self.clean_steps,
|
||||
'clean_step_index': 0},
|
||||
clean_step=self.clean_steps[0])
|
||||
mock_execute.return_value = return_state
|
||||
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_next_clean_step(task, self.next_clean_step_index)
|
||||
|
||||
node.refresh()
|
||||
|
||||
self.assertEqual(states.CLEANWAIT, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
self.assertEqual(self.clean_steps[1], node.clean_step)
|
||||
self.assertEqual(1, node.driver_internal_info['clean_step_index'])
|
||||
mock_execute.assert_called_once_with(
|
||||
mock.ANY, mock.ANY, self.clean_steps[1])
|
||||
|
||||
def test_do_next_clean_step_continue_from_last_cleaning(self):
|
||||
self._do_next_clean_step_continue_from_last_cleaning(states.CLEANWAIT)
|
||||
|
||||
def test_do_next_clean_step_manual_continue_from_last_cleaning(self):
|
||||
self._do_next_clean_step_continue_from_last_cleaning(states.CLEANWAIT,
|
||||
manual=True)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
|
||||
autospec=True)
|
||||
def _do_next_clean_step_last_step_noop(self, mock_execute, manual=False,
|
||||
retired=False):
|
||||
# Resume where last_step is the last cleaning step, should be noop
|
||||
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
|
||||
info = {'clean_steps': self.clean_steps,
|
||||
'clean_step_index': len(self.clean_steps) - 1}
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state,
|
||||
last_error=None,
|
||||
driver_internal_info=info,
|
||||
clean_step=self.clean_steps[-1],
|
||||
retired=retired)
|
||||
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_next_clean_step(task, None)
|
||||
|
||||
node.refresh()
|
||||
|
||||
# retired nodes move to manageable upon cleaning
|
||||
if retired:
|
||||
tgt_prov_state = states.MANAGEABLE
|
||||
|
||||
# Cleaning should be complete without calling additional steps
|
||||
self.assertEqual(tgt_prov_state, node.provision_state)
|
||||
self.assertEqual(states.NOSTATE, node.target_provision_state)
|
||||
self.assertEqual({}, node.clean_step)
|
||||
self.assertNotIn('clean_step_index', node.driver_internal_info)
|
||||
self.assertIsNone(node.driver_internal_info['clean_steps'])
|
||||
self.assertFalse(mock_execute.called)
|
||||
|
||||
def test__do_next_clean_step_automated_last_step_noop(self):
|
||||
self._do_next_clean_step_last_step_noop()
|
||||
|
||||
def test__do_next_clean_step_manual_last_step_noop(self):
|
||||
self._do_next_clean_step_last_step_noop(manual=True)
|
||||
|
||||
def test__do_next_clean_step_retired_last_step_change_tgt_state(self):
|
||||
self._do_next_clean_step_last_step_noop(retired=True)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_clean_step',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
|
||||
autospec=True)
|
||||
def _do_next_clean_step_all(self, mock_deploy_execute,
|
||||
mock_power_execute, manual=False):
|
||||
# Run all steps from start to finish (all synchronous)
|
||||
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state,
|
||||
last_error=None,
|
||||
driver_internal_info={'clean_steps': self.clean_steps,
|
||||
'clean_step_index': None},
|
||||
clean_step={})
|
||||
|
||||
def fake_deploy(conductor_obj, task, step):
|
||||
driver_internal_info = task.node.driver_internal_info
|
||||
driver_internal_info['goober'] = 'test'
|
||||
task.node.driver_internal_info = driver_internal_info
|
||||
task.node.save()
|
||||
|
||||
mock_deploy_execute.side_effect = fake_deploy
|
||||
mock_power_execute.return_value = None
|
||||
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_next_clean_step(task, 0)
|
||||
|
||||
node.refresh()
|
||||
|
||||
# Cleaning should be complete
|
||||
self.assertEqual(tgt_prov_state, node.provision_state)
|
||||
self.assertEqual(states.NOSTATE, node.target_provision_state)
|
||||
self.assertEqual({}, node.clean_step)
|
||||
self.assertNotIn('clean_step_index', node.driver_internal_info)
|
||||
self.assertEqual('test', node.driver_internal_info['goober'])
|
||||
self.assertIsNone(node.driver_internal_info['clean_steps'])
|
||||
mock_power_execute.assert_called_once_with(mock.ANY, mock.ANY,
|
||||
self.clean_steps[1])
|
||||
mock_deploy_execute.assert_has_calls(
|
||||
[mock.call(mock.ANY, mock.ANY, self.clean_steps[0]),
|
||||
mock.call(mock.ANY, mock.ANY, self.clean_steps[2])])
|
||||
|
||||
def test_do_next_clean_step_automated_all(self):
|
||||
self._do_next_clean_step_all()
|
||||
|
||||
def test_do_next_clean_step_manual_all(self):
|
||||
self._do_next_clean_step_all(manual=True)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
|
||||
autospec=True)
|
||||
@mock.patch.object(fake.FakeDeploy, 'tear_down_cleaning', autospec=True)
|
||||
def _do_next_clean_step_execute_fail(self, tear_mock, mock_execute,
|
||||
manual=False):
|
||||
# When a clean step fails, go to CLEANFAIL
|
||||
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state,
|
||||
last_error=None,
|
||||
driver_internal_info={'clean_steps': self.clean_steps,
|
||||
'clean_step_index': None},
|
||||
clean_step={})
|
||||
mock_execute.side_effect = Exception()
|
||||
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_next_clean_step(task, 0)
|
||||
tear_mock.assert_called_once_with(task.driver.deploy, task)
|
||||
|
||||
node.refresh()
|
||||
|
||||
# Make sure we go to CLEANFAIL, clear clean_steps
|
||||
self.assertEqual(states.CLEANFAIL, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
self.assertEqual({}, node.clean_step)
|
||||
self.assertNotIn('clean_step_index', node.driver_internal_info)
|
||||
self.assertIsNotNone(node.last_error)
|
||||
self.assertTrue(node.maintenance)
|
||||
mock_execute.assert_called_once_with(
|
||||
mock.ANY, mock.ANY, self.clean_steps[0])
|
||||
|
||||
def test__do_next_clean_step_automated_execute_fail(self):
|
||||
self._do_next_clean_step_execute_fail()
|
||||
|
||||
def test__do_next_clean_step_manual_execute_fail(self):
|
||||
self._do_next_clean_step_execute_fail(manual=True)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
|
||||
autospec=True)
|
||||
def test_do_next_clean_step_oob_reboot(self, mock_execute):
|
||||
# When a clean step fails, go to CLEANWAIT
|
||||
tgt_prov_state = states.MANAGEABLE
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state,
|
||||
last_error=None,
|
||||
driver_internal_info={'clean_steps': self.clean_steps,
|
||||
'clean_step_index': None,
|
||||
'cleaning_reboot': True},
|
||||
clean_step={})
|
||||
mock_execute.side_effect = exception.AgentConnectionFailed(
|
||||
reason='failed')
|
||||
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_next_clean_step(task, 0)
|
||||
|
||||
node.refresh()
|
||||
|
||||
# Make sure we go to CLEANWAIT
|
||||
self.assertEqual(states.CLEANWAIT, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
self.assertEqual(self.clean_steps[0], node.clean_step)
|
||||
self.assertEqual(0, node.driver_internal_info['clean_step_index'])
|
||||
self.assertFalse(node.driver_internal_info['skip_current_clean_step'])
|
||||
mock_execute.assert_called_once_with(
|
||||
mock.ANY, mock.ANY, self.clean_steps[0])
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
|
||||
autospec=True)
|
||||
def test_do_next_clean_step_oob_reboot_last_step(self, mock_execute):
|
||||
# Resume where last_step is the last cleaning step
|
||||
tgt_prov_state = states.MANAGEABLE
|
||||
info = {'clean_steps': self.clean_steps,
|
||||
'cleaning_reboot': True,
|
||||
'clean_step_index': len(self.clean_steps) - 1}
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state,
|
||||
last_error=None,
|
||||
driver_internal_info=info,
|
||||
clean_step=self.clean_steps[-1])
|
||||
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_next_clean_step(task, None)
|
||||
|
||||
node.refresh()
|
||||
|
||||
# Cleaning should be complete without calling additional steps
|
||||
self.assertEqual(tgt_prov_state, node.provision_state)
|
||||
self.assertEqual(states.NOSTATE, node.target_provision_state)
|
||||
self.assertEqual({}, node.clean_step)
|
||||
self.assertNotIn('clean_step_index', node.driver_internal_info)
|
||||
self.assertNotIn('cleaning_reboot', node.driver_internal_info)
|
||||
self.assertIsNone(node.driver_internal_info['clean_steps'])
|
||||
self.assertFalse(mock_execute.called)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
|
||||
autospec=True)
|
||||
@mock.patch.object(fake.FakeDeploy, 'tear_down_cleaning', autospec=True)
|
||||
def test_do_next_clean_step_oob_reboot_fail(self, tear_mock,
|
||||
mock_execute):
|
||||
# When a clean step fails with no reboot requested go to CLEANFAIL
|
||||
tgt_prov_state = states.MANAGEABLE
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state,
|
||||
last_error=None,
|
||||
driver_internal_info={'clean_steps': self.clean_steps,
|
||||
'clean_step_index': None},
|
||||
clean_step={})
|
||||
mock_execute.side_effect = exception.AgentConnectionFailed(
|
||||
reason='failed')
|
||||
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_next_clean_step(task, 0)
|
||||
tear_mock.assert_called_once_with(task.driver.deploy, task)
|
||||
|
||||
node.refresh()
|
||||
|
||||
# Make sure we go to CLEANFAIL, clear clean_steps
|
||||
self.assertEqual(states.CLEANFAIL, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
self.assertEqual({}, node.clean_step)
|
||||
self.assertNotIn('clean_step_index', node.driver_internal_info)
|
||||
self.assertNotIn('skip_current_clean_step', node.driver_internal_info)
|
||||
self.assertIsNotNone(node.last_error)
|
||||
self.assertTrue(node.maintenance)
|
||||
mock_execute.assert_called_once_with(
|
||||
mock.ANY, mock.ANY, self.clean_steps[0])
|
||||
|
||||
@mock.patch.object(cleaning, 'LOG', autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_clean_step',
|
||||
autospec=True)
|
||||
@mock.patch.object(fake.FakeDeploy, 'tear_down_cleaning', autospec=True)
|
||||
def _do_next_clean_step_fail_in_tear_down_cleaning(
|
||||
self, tear_mock, power_exec_mock, deploy_exec_mock, log_mock,
|
||||
manual=True):
|
||||
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state,
|
||||
last_error=None,
|
||||
driver_internal_info={'clean_steps': self.clean_steps,
|
||||
'clean_step_index': None},
|
||||
clean_step={})
|
||||
|
||||
deploy_exec_mock.return_value = None
|
||||
power_exec_mock.return_value = None
|
||||
tear_mock.side_effect = Exception('boom')
|
||||
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_next_clean_step(task, 0)
|
||||
|
||||
node.refresh()
|
||||
|
||||
# Make sure we go to CLEANFAIL, clear clean_steps
|
||||
self.assertEqual(states.CLEANFAIL, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
self.assertEqual({}, node.clean_step)
|
||||
self.assertNotIn('clean_step_index', node.driver_internal_info)
|
||||
self.assertIsNotNone(node.last_error)
|
||||
self.assertEqual(1, tear_mock.call_count)
|
||||
self.assertTrue(node.maintenance)
|
||||
deploy_exec_calls = [
|
||||
mock.call(mock.ANY, mock.ANY, self.clean_steps[0]),
|
||||
mock.call(mock.ANY, mock.ANY, self.clean_steps[2]),
|
||||
]
|
||||
self.assertEqual(deploy_exec_calls, deploy_exec_mock.call_args_list)
|
||||
|
||||
power_exec_calls = [
|
||||
mock.call(mock.ANY, mock.ANY, self.clean_steps[1]),
|
||||
]
|
||||
self.assertEqual(power_exec_calls, power_exec_mock.call_args_list)
|
||||
log_mock.exception.assert_called_once_with(
|
||||
'Failed to tear down from cleaning for node {}, reason: boom'
|
||||
.format(node.uuid))
|
||||
|
||||
def test__do_next_clean_step_automated_fail_in_tear_down_cleaning(self):
|
||||
self._do_next_clean_step_fail_in_tear_down_cleaning()
|
||||
|
||||
def test__do_next_clean_step_manual_fail_in_tear_down_cleaning(self):
|
||||
self._do_next_clean_step_fail_in_tear_down_cleaning(manual=True)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
|
||||
autospec=True)
|
||||
def _do_next_clean_step_no_steps(self, mock_execute, manual=False,
|
||||
fast_track=False):
|
||||
if fast_track:
|
||||
self.config(fast_track=True, group='deploy')
|
||||
|
||||
for info in ({'clean_steps': None, 'clean_step_index': None,
|
||||
'agent_url': 'test-url'},
|
||||
{'clean_steps': None, 'agent_url': 'test-url'}):
|
||||
# Resume where there are no steps, should be a noop
|
||||
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
uuid=uuidutils.generate_uuid(),
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state,
|
||||
last_error=None,
|
||||
driver_internal_info=info,
|
||||
clean_step={})
|
||||
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_next_clean_step(task, None)
|
||||
|
||||
node.refresh()
|
||||
|
||||
# Cleaning should be complete without calling additional steps
|
||||
self.assertEqual(tgt_prov_state, node.provision_state)
|
||||
self.assertEqual(states.NOSTATE, node.target_provision_state)
|
||||
self.assertEqual({}, node.clean_step)
|
||||
self.assertNotIn('clean_step_index', node.driver_internal_info)
|
||||
self.assertFalse(mock_execute.called)
|
||||
if fast_track:
|
||||
self.assertEqual('test-url',
|
||||
node.driver_internal_info.get('agent_url'))
|
||||
else:
|
||||
self.assertNotIn('agent_url', node.driver_internal_info)
|
||||
mock_execute.reset_mock()
|
||||
|
||||
def test__do_next_clean_step_automated_no_steps(self):
|
||||
self._do_next_clean_step_no_steps()
|
||||
|
||||
def test__do_next_clean_step_manual_no_steps(self):
|
||||
self._do_next_clean_step_no_steps(manual=True)
|
||||
|
||||
def test__do_next_clean_step_fast_track(self):
|
||||
self._do_next_clean_step_no_steps(fast_track=True)
|
||||
|
||||
@mock.patch('ironic.drivers.modules.fake.FakePower.execute_clean_step',
|
||||
autospec=True)
|
||||
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.execute_clean_step',
|
||||
autospec=True)
|
||||
def _do_next_clean_step_bad_step_return_value(
|
||||
self, deploy_exec_mock, power_exec_mock, manual=False):
|
||||
# When a clean step fails, go to CLEANFAIL
|
||||
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANING,
|
||||
target_provision_state=tgt_prov_state,
|
||||
last_error=None,
|
||||
driver_internal_info={'clean_steps': self.clean_steps,
|
||||
'clean_step_index': None},
|
||||
clean_step={})
|
||||
deploy_exec_mock.return_value = "foo"
|
||||
|
||||
with task_manager.acquire(
|
||||
self.context, node.uuid, shared=False) as task:
|
||||
cleaning.do_next_clean_step(task, 0)
|
||||
|
||||
node.refresh()
|
||||
|
||||
# Make sure we go to CLEANFAIL, clear clean_steps
|
||||
self.assertEqual(states.CLEANFAIL, node.provision_state)
|
||||
self.assertEqual(tgt_prov_state, node.target_provision_state)
|
||||
self.assertEqual({}, node.clean_step)
|
||||
self.assertNotIn('clean_step_index', node.driver_internal_info)
|
||||
self.assertIsNotNone(node.last_error)
|
||||
self.assertTrue(node.maintenance)
|
||||
deploy_exec_mock.assert_called_once_with(mock.ANY, mock.ANY,
|
||||
self.clean_steps[0])
|
||||
# Make sure we don't execute any other step and return
|
||||
self.assertFalse(power_exec_mock.called)
|
||||
|
||||
def test__do_next_clean_step_automated_bad_step_return_value(self):
|
||||
self._do_next_clean_step_bad_step_return_value()
|
||||
|
||||
def test__do_next_clean_step_manual_bad_step_return_value(self):
|
||||
self._do_next_clean_step_bad_step_return_value(manual=True)
|
||||
|
||||
|
||||
class DoNodeCleanAbortTestCase(db_base.DbTestCase):
|
||||
@mock.patch.object(fake.FakeDeploy, 'tear_down_cleaning', autospec=True)
|
||||
def _test__do_node_clean_abort(self, step_name, tear_mock):
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANFAIL,
|
||||
target_provision_state=states.AVAILABLE,
|
||||
clean_step={'step': 'foo', 'abortable': True},
|
||||
driver_internal_info={
|
||||
'clean_step_index': 2,
|
||||
'cleaning_reboot': True,
|
||||
'cleaning_polling': True,
|
||||
'skip_current_clean_step': True})
|
||||
|
||||
with task_manager.acquire(self.context, node.uuid) as task:
|
||||
cleaning.do_node_clean_abort(task, step_name=step_name)
|
||||
self.assertIsNotNone(task.node.last_error)
|
||||
tear_mock.assert_called_once_with(task.driver.deploy, task)
|
||||
if step_name:
|
||||
self.assertIn(step_name, task.node.last_error)
|
||||
# assert node's clean_step and metadata was cleaned up
|
||||
self.assertEqual({}, task.node.clean_step)
|
||||
self.assertNotIn('clean_step_index',
|
||||
task.node.driver_internal_info)
|
||||
self.assertNotIn('cleaning_reboot',
|
||||
task.node.driver_internal_info)
|
||||
self.assertNotIn('cleaning_polling',
|
||||
task.node.driver_internal_info)
|
||||
self.assertNotIn('skip_current_clean_step',
|
||||
task.node.driver_internal_info)
|
||||
|
||||
def test__do_node_clean_abort(self):
|
||||
self._test__do_node_clean_abort(None)
|
||||
|
||||
def test__do_node_clean_abort_with_step_name(self):
|
||||
self._test__do_node_clean_abort('foo')
|
||||
|
||||
@mock.patch.object(fake.FakeDeploy, 'tear_down_cleaning', autospec=True)
|
||||
def test__do_node_clean_abort_tear_down_fail(self, tear_mock):
|
||||
tear_mock.side_effect = Exception('Surprise')
|
||||
|
||||
node = obj_utils.create_test_node(
|
||||
self.context, driver='fake-hardware',
|
||||
provision_state=states.CLEANFAIL,
|
||||
target_provision_state=states.AVAILABLE,
|
||||
clean_step={'step': 'foo', 'abortable': True})
|
||||
|
||||
with task_manager.acquire(self.context, node.uuid) as task:
|
||||
cleaning.do_node_clean_abort(task)
|
||||
tear_mock.assert_called_once_with(task.driver.deploy, task)
|
||||
self.assertIsNotNone(task.node.last_error)
|
||||
self.assertIsNotNone(task.node.maintenance_reason)
|
||||
self.assertTrue(task.node.maintenance)
|
||||
self.assertEqual('clean failure', task.node.fault)
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user