DRAC : clear_job_queue clean step to fix pending bios config jobs
If pending bios config job for changing the boot mode is in job queue on an overcloud node, then ironic will abandon the introspection attempt for the node, which will cause overall introspection to fail. Added clear_job_queue cleaning step in the iDRAC driver that clears the Lifecycle Controller job queue even if there is pending bios config job or any other pending job in job queue. Change-Id: I35e4c61dd44384429224dee6de9827d8debe9cf9 Story: 2006322 Task: 36053
This commit is contained in:
Pradip Kadam
parent
af61985d03
commit
dd9c5fcedf
@ -80,6 +80,9 @@ _NON_PERSISTENT_BOOT_MODE = 'OneTime'
|
||||
# Clear job id's constant
|
||||
_CLEAR_JOB_IDS = 'JID_CLEARALL_FORCE'
|
||||
|
||||
# BIOS pending job constant
|
||||
_BIOS_JOB_NAME = 'BIOS.Setup.1-1'
|
||||
|
||||
|
||||
def _get_boot_device(node, drac_boot_devices=None):
|
||||
client = drac_common.get_drac_client(node)
|
||||
@ -187,10 +190,19 @@ def set_boot_device(node, device, persistent=False):
|
||||
:raises: DracOperationError on an error from python-dracclient.
|
||||
"""
|
||||
|
||||
drac_job.validate_job_queue(node)
|
||||
|
||||
client = drac_common.get_drac_client(node)
|
||||
|
||||
# If pending BIOS job found in job queue, we need to clear that job
|
||||
# before executing cleaning step of management interface.
|
||||
# Otherwise, pending BIOS config job can cause creating new config jobs
|
||||
# to fail.
|
||||
unfinished_jobs = drac_job.list_unfinished_jobs(node)
|
||||
if unfinished_jobs:
|
||||
unfinished_bios_jobs = [job.id for job in unfinished_jobs if
|
||||
_BIOS_JOB_NAME in job.name]
|
||||
if unfinished_bios_jobs:
|
||||
client.delete_jobs(job_ids=unfinished_bios_jobs)
|
||||
|
||||
try:
|
||||
drac_boot_devices = client.list_boot_devices()
|
||||
|
||||
@ -414,3 +426,23 @@ class DracManagement(base.ManagementInterface):
|
||||
client = drac_common.get_drac_client(node)
|
||||
client.reset_idrac(force=True, wait=True)
|
||||
client.delete_jobs(job_ids=[_CLEAR_JOB_IDS])
|
||||
|
||||
@METRICS.timer('DracManagement.clear_job_queue')
|
||||
@base.clean_step(priority=0)
|
||||
def clear_job_queue(self, task):
|
||||
"""Clear the job queue.
|
||||
|
||||
:param task: a TaskManager instance containing the node to act on.
|
||||
:returns: None if it is completed.
|
||||
:raises: DracOperationError on an error from python-dracclient.
|
||||
"""
|
||||
try:
|
||||
node = task.node
|
||||
|
||||
client = drac_common.get_drac_client(node)
|
||||
client.delete_jobs(job_ids=[_CLEAR_JOB_IDS])
|
||||
except drac_exceptions.BaseClientException as exc:
|
||||
LOG.error('DRAC driver failed to clear the job queue for node '
|
||||
'%(node_uuid)s. Reason: %(error)s.',
|
||||
{'node_uuid': node.uuid, 'error': exc})
|
||||
raise exception.DracOperationError(error=exc)
|
||||
|
||||
@ -248,15 +248,17 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
spec_set=True, autospec=True)
|
||||
@mock.patch.object(drac_mgmt, '_get_boot_device', spec_set=True,
|
||||
autospec=True)
|
||||
@mock.patch.object(drac_job, 'validate_job_queue', spec_set=True,
|
||||
@mock.patch.object(drac_job, 'list_unfinished_jobs', spec_set=True,
|
||||
autospec=True)
|
||||
def test_set_boot_device(self, mock_validate_job_queue,
|
||||
def test_set_boot_device(self, mock_list_unfinished_jobs,
|
||||
mock__get_boot_device,
|
||||
mock__get_next_persistent_boot_mode,
|
||||
mock_get_drac_client):
|
||||
mock_client = mock.Mock()
|
||||
mock_get_drac_client.return_value = mock_client
|
||||
mock_client.list_boot_devices.return_value = self.boot_devices['IPL']
|
||||
mock_list_unfinished_jobs.return_value = []
|
||||
|
||||
mock_job = mock.Mock()
|
||||
mock_job.status = "Scheduled"
|
||||
mock_client.get_job.return_value = mock_job
|
||||
@ -269,7 +271,7 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
boot_device = drac_mgmt.set_boot_device(
|
||||
self.node, ironic.common.boot_devices.PXE, persistent=False)
|
||||
|
||||
mock_validate_job_queue.assert_called_once_with(self.node)
|
||||
mock_list_unfinished_jobs.assert_called_once_with(self.node)
|
||||
mock_client.change_boot_device_order.assert_called_once_with(
|
||||
'OneTime', 'BIOS.Setup.1-1#BootSeq#NIC.Embedded.1-1-1')
|
||||
self.assertEqual(0, mock_client.set_bios_settings.call_count)
|
||||
@ -279,10 +281,10 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
spec_set=True, autospec=True)
|
||||
@mock.patch.object(drac_mgmt, '_get_boot_device', spec_set=True,
|
||||
autospec=True)
|
||||
@mock.patch.object(drac_job, 'validate_job_queue', spec_set=True,
|
||||
@mock.patch.object(drac_job, 'list_unfinished_jobs', spec_set=True,
|
||||
autospec=True)
|
||||
def test_set_boot_device_called_with_no_change(
|
||||
self, mock_validate_job_queue, mock__get_boot_device,
|
||||
self, mock_list_unfinished_jobs, mock__get_boot_device,
|
||||
mock__get_next_persistent_boot_mode, mock_get_drac_client):
|
||||
mock_client = mock.Mock()
|
||||
mock_get_drac_client.return_value = mock_client
|
||||
@ -291,11 +293,12 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
'persistent': True}
|
||||
mock__get_boot_device.return_value = boot_device
|
||||
mock__get_next_persistent_boot_mode.return_value = 'IPL'
|
||||
mock_list_unfinished_jobs.return_value = []
|
||||
|
||||
boot_device = drac_mgmt.set_boot_device(
|
||||
self.node, ironic.common.boot_devices.PXE, persistent=True)
|
||||
|
||||
mock_validate_job_queue.assert_called_once_with(self.node)
|
||||
mock_list_unfinished_jobs.assert_called_once_with(self.node)
|
||||
self.assertEqual(0, mock_client.change_boot_device_order.call_count)
|
||||
self.assertEqual(0, mock_client.set_bios_settings.call_count)
|
||||
self.assertEqual(0, mock_client.commit_pending_bios_changes.call_count)
|
||||
@ -308,17 +311,18 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
spec_set=True, autospec=True)
|
||||
@mock.patch.object(drac_mgmt, '_get_boot_device', spec_set=True,
|
||||
autospec=True)
|
||||
@mock.patch.object(drac_job, 'validate_job_queue', spec_set=True,
|
||||
@mock.patch.object(drac_job, 'list_unfinished_jobs', spec_set=True,
|
||||
autospec=True)
|
||||
def test_set_boot_device_called_with_no_drac_boot_device(
|
||||
self, mock_validate_job_queue, mock__get_boot_device,
|
||||
mock__get_next_persistent_boot_mode,
|
||||
self, mock_list_unfinished_jobs,
|
||||
mock__get_boot_device, mock__get_next_persistent_boot_mode,
|
||||
mock__is_boot_order_flexibly_programmable,
|
||||
mock__flexibly_program_boot_order,
|
||||
mock_get_drac_client):
|
||||
mock_client = mock.Mock()
|
||||
mock_get_drac_client.return_value = mock_client
|
||||
mock_client.list_boot_devices.return_value = self.boot_devices['UEFI']
|
||||
mock_list_unfinished_jobs.return_value = []
|
||||
|
||||
mock_job = mock.Mock()
|
||||
mock_job.status = "Scheduled"
|
||||
@ -354,7 +358,7 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
drac_mgmt.set_boot_device(self.node, ironic.common.boot_devices.DISK,
|
||||
persistent=True)
|
||||
|
||||
mock_validate_job_queue.assert_called_once_with(self.node)
|
||||
mock_list_unfinished_jobs.assert_called_once_with(self.node)
|
||||
self.assertEqual(0, mock_client.change_boot_device_order.call_count)
|
||||
mock_client.set_bios_settings.assert_called_once_with(
|
||||
flexibly_program_settings)
|
||||
@ -366,15 +370,16 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
spec_set=True, autospec=True)
|
||||
@mock.patch.object(drac_mgmt, '_get_boot_device', spec_set=True,
|
||||
autospec=True)
|
||||
@mock.patch.object(drac_job, 'validate_job_queue', spec_set=True,
|
||||
@mock.patch.object(drac_job, 'list_unfinished_jobs', spec_set=True,
|
||||
autospec=True)
|
||||
def test_set_boot_device_called_with_not_flexibly_programmable(
|
||||
self, mock_validate_job_queue, mock__get_boot_device,
|
||||
mock__get_next_persistent_boot_mode,
|
||||
self, mock_list_unfinished_jobs,
|
||||
mock__get_boot_device, mock__get_next_persistent_boot_mode,
|
||||
mock__is_boot_order_flexibly_programmable,
|
||||
mock_get_drac_client):
|
||||
mock_client = mock.Mock()
|
||||
mock_get_drac_client.return_value = mock_client
|
||||
mock_list_unfinished_jobs.return_value = []
|
||||
mock_client.list_boot_devices.return_value = self.boot_devices['UEFI']
|
||||
boot_device = {'boot_device': ironic.common.boot_devices.PXE,
|
||||
'persistent': False}
|
||||
@ -386,7 +391,7 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
drac_mgmt.set_boot_device, self.node,
|
||||
ironic.common.boot_devices.CDROM, persistent=False)
|
||||
|
||||
mock_validate_job_queue.assert_called_once_with(self.node)
|
||||
mock_list_unfinished_jobs.assert_called_once_with(self.node)
|
||||
self.assertEqual(0, mock_client.change_boot_device_order.call_count)
|
||||
self.assertEqual(0, mock_client.set_bios_settings.call_count)
|
||||
self.assertEqual(0, mock_client.commit_pending_bios_changes.call_count)
|
||||
@ -397,15 +402,16 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
spec_set=True, autospec=True)
|
||||
@mock.patch.object(drac_mgmt, '_get_boot_device', spec_set=True,
|
||||
autospec=True)
|
||||
@mock.patch.object(drac_job, 'validate_job_queue', spec_set=True,
|
||||
@mock.patch.object(drac_job, 'list_unfinished_jobs', spec_set=True,
|
||||
autospec=True)
|
||||
def test_set_boot_device_called_with_unknown_boot_mode(
|
||||
self, mock_validate_job_queue, mock__get_boot_device,
|
||||
self, mock_list_unfinished_jobs, mock__get_boot_device,
|
||||
mock__get_next_persistent_boot_mode,
|
||||
mock__is_boot_order_flexibly_programmable,
|
||||
mock_get_drac_client):
|
||||
mock_client = mock.Mock()
|
||||
mock_get_drac_client.return_value = mock_client
|
||||
|
||||
mock_client.list_boot_devices.return_value = self.boot_devices['UEFI']
|
||||
boot_device = {'boot_device': ironic.common.boot_devices.PXE,
|
||||
'persistent': False}
|
||||
@ -426,29 +432,11 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
values=s) for s in settings}
|
||||
mock_client.list_bios_settings.return_value = bios_settings
|
||||
mock__is_boot_order_flexibly_programmable.return_value = True
|
||||
|
||||
mock_list_unfinished_jobs.return_value = []
|
||||
self.assertRaises(exception.DracOperationError,
|
||||
drac_mgmt.set_boot_device, self.node,
|
||||
ironic.common.boot_devices.DISK, persistent=True)
|
||||
|
||||
mock_validate_job_queue.assert_called_once_with(self.node)
|
||||
self.assertEqual(0, mock_client.change_boot_device_order.call_count)
|
||||
self.assertEqual(0, mock_client.set_bios_settings.call_count)
|
||||
self.assertEqual(0, mock_client.commit_pending_bios_changes.call_count)
|
||||
|
||||
@mock.patch.object(drac_job, 'validate_job_queue', spec_set=True,
|
||||
autospec=True)
|
||||
def test_set_boot_device_with_invalid_job_queue(
|
||||
self, mock_validate_job_queue, mock_get_drac_client):
|
||||
mock_client = mock.Mock()
|
||||
mock_get_drac_client.return_value = mock_client
|
||||
mock_validate_job_queue.side_effect = exception.DracOperationError(
|
||||
'boom')
|
||||
|
||||
self.assertRaises(exception.DracOperationError,
|
||||
drac_mgmt.set_boot_device, self.node,
|
||||
ironic.common.boot_devices.PXE, persistent=True)
|
||||
|
||||
mock_list_unfinished_jobs.assert_called_once_with(self.node)
|
||||
self.assertEqual(0, mock_client.change_boot_device_order.call_count)
|
||||
self.assertEqual(0, mock_client.set_bios_settings.call_count)
|
||||
self.assertEqual(0, mock_client.commit_pending_bios_changes.call_count)
|
||||
@ -459,11 +447,11 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
spec_set=True, autospec=True)
|
||||
@mock.patch.object(drac_mgmt, '_get_boot_device', spec_set=True,
|
||||
autospec=True)
|
||||
@mock.patch.object(drac_job, 'validate_job_queue', spec_set=True,
|
||||
@mock.patch.object(drac_job, 'list_unfinished_jobs', spec_set=True,
|
||||
autospec=True)
|
||||
def test_set_boot_device_job_not_scheduled(
|
||||
self,
|
||||
mock_validate_job_queue,
|
||||
mock_list_unfinished_jobs,
|
||||
mock__get_boot_device,
|
||||
mock__get_next_persistent_boot_mode,
|
||||
mock_sleep,
|
||||
@ -471,6 +459,7 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
mock_get_drac_client):
|
||||
mock_client = mock.Mock()
|
||||
mock_get_drac_client.return_value = mock_client
|
||||
mock_list_unfinished_jobs.return_value = []
|
||||
mock_client.list_boot_devices.return_value = self.boot_devices['IPL']
|
||||
mock_job = mock.Mock()
|
||||
mock_job.status = "New"
|
||||
@ -486,6 +475,107 @@ class DracManagementInternalMethodsTestCase(test_utils.BaseDracTest):
|
||||
drac_mgmt.set_boot_device, self.node,
|
||||
ironic.common.boot_devices.PXE,
|
||||
persistent=True)
|
||||
mock_list_unfinished_jobs.assert_called_once_with(self.node)
|
||||
|
||||
@mock.patch.object(drac_job, 'list_unfinished_jobs', spec_set=True,
|
||||
autospec=True)
|
||||
def test_set_boot_device_with_list_unfinished_jobs_fail(
|
||||
self, mock_list_unfinished_jobs, mock_get_drac_client):
|
||||
mock_client = mock.Mock()
|
||||
mock_get_drac_client.return_value = mock_client
|
||||
|
||||
mock_list_unfinished_jobs.side_effect = exception.DracOperationError(
|
||||
'boom')
|
||||
|
||||
self.assertRaises(exception.DracOperationError,
|
||||
drac_mgmt.set_boot_device, self.node,
|
||||
ironic.common.boot_devices.PXE, persistent=True)
|
||||
|
||||
self.assertEqual(0, mock_client.change_boot_device_order.call_count)
|
||||
self.assertEqual(0, mock_client.set_bios_settings.call_count)
|
||||
self.assertEqual(0, mock_client.commit_pending_bios_changes.call_count)
|
||||
|
||||
@mock.patch.object(drac_job, 'list_unfinished_jobs', spec_set=True,
|
||||
autospec=True)
|
||||
@mock.patch.object(drac_mgmt, '_get_boot_device', spec_set=True,
|
||||
autospec=True)
|
||||
@mock.patch.object(drac_mgmt, '_get_next_persistent_boot_mode',
|
||||
spec_set=True, autospec=True)
|
||||
def test_set_boot_device_with_list_unfinished_jobs(
|
||||
self, mock__get_next_persistent_boot_mode, mock__get_boot_device,
|
||||
mock_list_unfinished_jobs, mock_get_drac_client):
|
||||
mock_client = mock.Mock()
|
||||
mock_get_drac_client.return_value = mock_client
|
||||
|
||||
bios_job_dict = {
|
||||
'id': 'JID_602553293345',
|
||||
'name': 'ConfigBIOS:BIOS.Setup.1-1',
|
||||
'start_time': 'TIME_NOW',
|
||||
'until_time': 'TIME_NA',
|
||||
'message': 'Task successfully scheduled.',
|
||||
'status': 'Scheduled',
|
||||
'percent_complete': 0}
|
||||
bios_job = test_utils.make_job(bios_job_dict)
|
||||
|
||||
mock_list_unfinished_jobs.return_value = [bios_job]
|
||||
mock_client.list_boot_devices.return_value = self.boot_devices['IPL']
|
||||
boot_device = {'boot_device': ironic.common.boot_devices.DISK,
|
||||
'persistent': True}
|
||||
|
||||
mock__get_boot_device.return_value = boot_device
|
||||
mock__get_next_persistent_boot_mode.return_value = 'IPL'
|
||||
|
||||
drac_mgmt.set_boot_device(self.node, ironic.common.boot_devices.DISK,
|
||||
persistent=True)
|
||||
mock_list_unfinished_jobs.assert_called_once_with(self.node)
|
||||
mock_client.delete_jobs.assert_called_once_with(
|
||||
job_ids=['JID_602553293345'])
|
||||
|
||||
@mock.patch.object(drac_job, 'list_unfinished_jobs', spec_set=True,
|
||||
autospec=True)
|
||||
@mock.patch.object(drac_mgmt, '_get_boot_device', spec_set=True,
|
||||
autospec=True)
|
||||
@mock.patch.object(drac_mgmt, '_get_next_persistent_boot_mode',
|
||||
spec_set=True, autospec=True)
|
||||
def test_set_boot_device_with_multiple_unfinished_jobs(
|
||||
self, mock__get_next_persistent_boot_mode, mock__get_boot_device,
|
||||
mock_list_unfinished_jobs, mock_get_drac_client):
|
||||
mock_client = mock.Mock()
|
||||
mock_get_drac_client.return_value = mock_client
|
||||
|
||||
job_dict = {
|
||||
'id': 'JID_602553293345',
|
||||
'name': 'Config:RAID:RAID.Integrated.1-1',
|
||||
'start_time': 'TIME_NOW',
|
||||
'until_time': 'TIME_NA',
|
||||
'message': 'Task successfully scheduled.',
|
||||
'status': 'Scheduled',
|
||||
'percent_complete': 0}
|
||||
job = test_utils.make_job(job_dict)
|
||||
|
||||
bios_job_dict = {
|
||||
'id': 'JID_602553293346',
|
||||
'name': 'ConfigBIOS:BIOS.Setup.1-1',
|
||||
'start_time': 'TIME_NOW',
|
||||
'until_time': 'TIME_NA',
|
||||
'message': 'Task successfully scheduled.',
|
||||
'status': 'Scheduled',
|
||||
'percent_complete': 0}
|
||||
bios_job = test_utils.make_job(bios_job_dict)
|
||||
|
||||
mock_list_unfinished_jobs.return_value = [job, bios_job]
|
||||
mock_client.list_boot_devices.return_value = self.boot_devices['IPL']
|
||||
boot_device = {'boot_device': ironic.common.boot_devices.DISK,
|
||||
'persistent': True}
|
||||
|
||||
mock__get_boot_device.return_value = boot_device
|
||||
mock__get_next_persistent_boot_mode.return_value = 'IPL'
|
||||
|
||||
drac_mgmt.set_boot_device(self.node, ironic.common.boot_devices.DISK,
|
||||
persistent=True)
|
||||
mock_list_unfinished_jobs.assert_called_once_with(self.node)
|
||||
mock_client.delete_jobs.assert_called_once_with(
|
||||
job_ids=['JID_602553293346'])
|
||||
|
||||
|
||||
@mock.patch.object(drac_common, 'get_drac_client', spec_set=True,
|
||||
@ -600,3 +690,15 @@ class DracManagementTestCase(test_utils.BaseDracTest):
|
||||
job_ids=['JID_CLEARALL_FORCE'])
|
||||
|
||||
self.assertIsNone(return_value)
|
||||
|
||||
def test_clear_job_queue(self, mock_get_drac_client):
|
||||
mock_client = mock.Mock()
|
||||
mock_get_drac_client.return_value = mock_client
|
||||
|
||||
with task_manager.acquire(self.context, self.node.uuid,
|
||||
shared=False) as task:
|
||||
return_value = task.driver.management.clear_job_queue(task)
|
||||
mock_client.delete_jobs.assert_called_once_with(
|
||||
job_ids=['JID_CLEARALL_FORCE'])
|
||||
|
||||
self.assertIsNone(return_value)
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
---
|
||||
features:
|
||||
- |
|
||||
Adds ``clear_job_queue`` cleaning step to hardware type ``idrac``.
|
||||
``clear_job_queue`` cleaning step clears the Lifecycle Controller
|
||||
job queue including any pending jobs.
|
||||
fixes:
|
||||
- |
|
||||
Fixes an issue where if there is a pending bios config job in job queue,
|
||||
then ironic will abandon the introspection attempt for the node,
|
||||
which will cause overall introspection to fail.
|
||||
Loading…
Reference in New Issue
Block a user