From 1dda97c783653aef638113cb1faa250836ed99e1 Mon Sep 17 00:00:00 2001 From: Jacob Anders Date: Tue, 21 Jun 2022 13:18:12 +1000 Subject: [PATCH] Prevent clear_job_queue and reset_idrac failures on older iDRACs Currently, clear_job_queue and reset_idrac steps are only supported on idrac-redfish driver on iDRAC9 hardware. However, Ironic still attempts to run these steps on iDRAC8 BMCs configured with idrac-redfish driver which results in verification failures. This change attempts to resolve it by catching the related exception, logging a warning and continuing verification. In case of cleaning, it still fails. Story: 2010091 Task: 45630 Change-Id: Icd8c5378469887962ff32eea2f38697c539f7e95 --- ironic/drivers/modules/drac/management.py | 42 +++++++-- .../drivers/modules/drac/test_management.py | 90 +++++++++++++++++++ ...eset-if-attr-missing-b2a2b609c906c6c4.yaml | 8 ++ 3 files changed, 132 insertions(+), 8 deletions(-) create mode 100644 releasenotes/notes/skip-clear-job-queue-idrac-reset-if-attr-missing-b2a2b609c906c6c4.yaml diff --git a/ironic/drivers/modules/drac/management.py b/ironic/drivers/modules/drac/management.py index df6942611f..ca250ae275 100644 --- a/ironic/drivers/modules/drac/management.py +++ b/ironic/drivers/modules/drac/management.py @@ -33,6 +33,7 @@ from ironic.common import boot_devices from ironic.common import exception from ironic.common.i18n import _ from ironic.common import molds +from ironic.common import states from ironic.conductor import periodics from ironic.conductor import task_manager from ironic.conductor import utils as manager_utils @@ -623,9 +624,22 @@ class DracRedfishManagement(redfish_management.RedfishManagement): on. :raises: RedfishError on an error. """ - drac_utils.execute_oem_manager_method( - task, 'clear job queue', - lambda m: m.job_service.delete_jobs(job_ids=['JID_CLEARALL'])) + try: + drac_utils.execute_oem_manager_method( + task, 'clear job queue', + lambda m: m.job_service.delete_jobs(job_ids=['JID_CLEARALL'])) + except exception.RedfishError as exc: + if "Oem/Dell/DellJobService is missing" in str(exc): + LOG.warning('iDRAC on node %(node)s does not support ' + 'clearing Lifecycle Controller job queue ' + 'using the idrac-redfish driver. ' + 'If using iDRAC9, consider upgrading firmware. ' + 'If using iDRAC8, consider switching to ' + 'idrac-wsman for management interface if ' + 'possible.', + {'node': task.node.uuid}) + if task.node.provision_state != states.VERIFYING: + raise @METRICS.timer('DracRedfishManagement.reset_idrac') @base.verify_step(priority=0) @@ -637,11 +651,23 @@ class DracRedfishManagement(redfish_management.RedfishManagement): on. :raises: RedfishError on an error. """ - drac_utils.execute_oem_manager_method( - task, 'reset iDRAC', lambda m: m.reset_idrac()) - redfish_utils.wait_until_get_system_ready(task.node) - LOG.info('Reset iDRAC for node %(node)s done', - {'node': task.node.uuid}) + try: + drac_utils.execute_oem_manager_method( + task, 'reset iDRAC', lambda m: m.reset_idrac()) + redfish_utils.wait_until_get_system_ready(task.node) + LOG.info('Reset iDRAC for node %(node)s done', + {'node': task.node.uuid}) + except exception.RedfishError as exc: + if "Oem/Dell/DelliDRACCardService is missing" in str(exc): + LOG.warning('iDRAC on node %(node)s does not support ' + 'iDRAC reset using the idrac-redfish driver. ' + 'If using iDRAC9, consider upgrading firmware. ' + 'If using iDRAC8, consider switching to ' + 'idrac-wsman for management interface if ' + 'possible.', + {'node': task.node.uuid}) + if task.node.provision_state != states.VERIFYING: + raise @METRICS.timer('DracRedfishManagement.known_good_state') @base.verify_step(priority=0) diff --git a/ironic/tests/unit/drivers/modules/drac/test_management.py b/ironic/tests/unit/drivers/modules/drac/test_management.py index ba6857504a..fa1ed61304 100644 --- a/ironic/tests/unit/drivers/modules/drac/test_management.py +++ b/ironic/tests/unit/drivers/modules/drac/test_management.py @@ -28,6 +28,7 @@ from oslo_utils import importutils import ironic.common.boot_devices from ironic.common import exception from ironic.common import molds +from ironic.common import states from ironic.conductor import periodics from ironic.conductor import task_manager from ironic.conductor import utils as manager_utils @@ -1438,6 +1439,48 @@ class DracRedfishManagementTestCase(test_utils.BaseDracTest): mock_manager_oem.job_service.delete_jobs.assert_called_once_with( job_ids=['JID_CLEARALL']) + @mock.patch.object(drac_mgmt, 'LOG', autospec=True) + @mock.patch.object(drac_utils, 'redfish_utils', autospec=True) + def test_clear_job_queue_missing_attr_verify_step(self, + mock_redfish_utils, + mock_log): + mock_system = mock_redfish_utils.get_system.return_value + mock_manager = mock.MagicMock() + mock_system.managers = [mock_manager] + mock_manager_oem = mock_manager.get_oem_extension.return_value + mock_manager_oem.job_service.delete_jobs.side_effect = ( + exception.RedfishError("Oem/Dell/DellJobService is missing")) + with task_manager.acquire(self.context, self.node.uuid, + shared=False) as task: + task.node.provision_state = states.VERIFYING + task.driver.management.clear_job_queue(task) + mock_log.warning.assert_called_once_with( + 'iDRAC on node %(node)s does not support ' + 'clearing Lifecycle Controller job queue ' + 'using the idrac-redfish driver. ' + 'If using iDRAC9, consider upgrading firmware. ' + 'If using iDRAC8, consider switching to ' + 'idrac-wsman for management interface if ' + 'possible.', + {'node': task.node.uuid}) + + @mock.patch.object(drac_mgmt, 'LOG', autospec=True) + @mock.patch.object(drac_utils, 'redfish_utils', autospec=True) + def test_clear_job_queue_missing_attr_clean_step(self, + mock_redfish_utils, + mock_log): + mock_system = mock_redfish_utils.get_system.return_value + mock_manager = mock.MagicMock() + mock_system.managers = [mock_manager] + mock_manager_oem = mock_manager.get_oem_extension.return_value + mock_manager_oem.job_service.delete_jobs.side_effect = ( + exception.RedfishError("Oem/Dell/DellJobService is missing")) + with task_manager.acquire(self.context, self.node.uuid, + shared=False) as task: + task.node.provision_state = states.CLEANING + self.assertRaises(ironic.common.exception.RedfishError, + task.driver.management.clear_job_queue, task) + @mock.patch.object(redfish_utils, 'wait_until_get_system_ready', autospec=True) @mock.patch.object(drac_utils, 'redfish_utils', autospec=True) @@ -1452,6 +1495,53 @@ class DracRedfishManagementTestCase(test_utils.BaseDracTest): task.driver.management.reset_idrac(task) mock_manager_oem.reset_idrac.assert_called_once_with() + @mock.patch.object(redfish_utils, 'wait_until_get_system_ready', + autospec=True) + @mock.patch.object(drac_mgmt, 'LOG', autospec=True) + @mock.patch.object(drac_utils, 'redfish_utils', autospec=True) + def test_reset_idrac_missing_attr_verify_step(self, + mock_redfish_utils, + mock_log, + mock_wait_system_ready): + mock_system = mock_redfish_utils.get_system.return_value + mock_manager = mock.MagicMock() + mock_system.managers = [mock_manager] + mock_manager_oem = mock_manager.get_oem_extension.return_value + mock_manager_oem.reset_idrac.side_effect = ( + exception.RedfishError("Oem/Dell/DelliDRACCardService is missing")) + with task_manager.acquire(self.context, self.node.uuid, + shared=False) as task: + task.node.provision_state = states.VERIFYING + task.driver.management.reset_idrac(task) + mock_log.warning.assert_called_once_with( + 'iDRAC on node %(node)s does not support ' + 'iDRAC reset using the idrac-redfish driver. ' + 'If using iDRAC9, consider upgrading firmware. ' + 'If using iDRAC8, consider switching to ' + 'idrac-wsman for management interface if ' + 'possible.', + {'node': task.node.uuid}) + + @mock.patch.object(redfish_utils, 'wait_until_get_system_ready', + autospec=True) + @mock.patch.object(drac_mgmt, 'LOG', autospec=True) + @mock.patch.object(drac_utils, 'redfish_utils', autospec=True) + def test_reset_idrac_missing_attr_clean_step(self, + mock_redfish_utils, + mock_log, + mock_wait_system_ready): + mock_system = mock_redfish_utils.get_system.return_value + mock_manager = mock.MagicMock() + mock_system.managers = [mock_manager] + mock_manager_oem = mock_manager.get_oem_extension.return_value + mock_manager_oem.reset_idrac.side_effect = ( + exception.RedfishError("Oem/Dell/DelliDRACCardService is missing")) + with task_manager.acquire(self.context, self.node.uuid, + shared=False) as task: + task.node.provision_state = states.CLEANING + self.assertRaises(ironic.common.exception.RedfishError, + task.driver.management.reset_idrac, task) + @mock.patch.object(redfish_utils, 'wait_until_get_system_ready', autospec=True) @mock.patch.object(drac_utils, 'redfish_utils', autospec=True) diff --git a/releasenotes/notes/skip-clear-job-queue-idrac-reset-if-attr-missing-b2a2b609c906c6c4.yaml b/releasenotes/notes/skip-clear-job-queue-idrac-reset-if-attr-missing-b2a2b609c906c6c4.yaml new file mode 100644 index 0000000000..df9bef955a --- /dev/null +++ b/releasenotes/notes/skip-clear-job-queue-idrac-reset-if-attr-missing-b2a2b609c906c6c4.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Resolved clear_job_queue and reset_idrac verify step failures which occur + when the functionality is not supported by the iDRAC. When this condition + is detected, the code in the step handles the exception and logs a warning + and completes successfully in case of verification steps but fails in case + of cleaning steps.