Prevent clear_job_queue and reset_idrac failures on older iDRACs

Currently, clear_job_queue and reset_idrac steps are only supported on
idrac-redfish driver on iDRAC9 hardware. However, Ironic still attempts
to run these steps on iDRAC8 BMCs configured with idrac-redfish driver
which results in verification failures. This change attempts to resolve
it by catching the related exception, logging a warning and continuing
verification. In case of cleaning, it still fails.

Story: 2010091
Task: 45630
Change-Id: Icd8c5378469887962ff32eea2f38697c539f7e95

ironic/drivers/modules/drac/management.py

@@ -33,6 +33,7 @@ from ironic.common import boot_devices
 from ironic.common import exception
 from ironic.common.i18n import _
 from ironic.common import molds
+from ironic.common import states
 from ironic.conductor import periodics
 from ironic.conductor import task_manager
 from ironic.conductor import utils as manager_utils
@@ -623,9 +624,22 @@ class DracRedfishManagement(redfish_management.RedfishManagement):
                      on.
         :raises: RedfishError on an error.
         """
-        drac_utils.execute_oem_manager_method(
+        try:
-            task, 'clear job queue',
+            drac_utils.execute_oem_manager_method(
-            lambda m: m.job_service.delete_jobs(job_ids=['JID_CLEARALL']))
+                task, 'clear job queue',
+                lambda m: m.job_service.delete_jobs(job_ids=['JID_CLEARALL']))
+        except exception.RedfishError as exc:
+            if "Oem/Dell/DellJobService is missing" in str(exc):
+                LOG.warning('iDRAC on node %(node)s does not support '
+                            'clearing Lifecycle Controller job queue '
+                            'using the idrac-redfish driver. '
+                            'If using iDRAC9, consider upgrading firmware. '
+                            'If using iDRAC8, consider switching to '
+                            'idrac-wsman for management interface if '
+                            'possible.',
+                            {'node': task.node.uuid})
+            if task.node.provision_state != states.VERIFYING:
+                raise
 
     @METRICS.timer('DracRedfishManagement.reset_idrac')
     @base.verify_step(priority=0)
@@ -637,11 +651,23 @@ class DracRedfishManagement(redfish_management.RedfishManagement):
                      on.
         :raises: RedfishError on an error.
         """
-        drac_utils.execute_oem_manager_method(
+        try:
-            task, 'reset iDRAC', lambda m: m.reset_idrac())
+            drac_utils.execute_oem_manager_method(
-        redfish_utils.wait_until_get_system_ready(task.node)
+                task, 'reset iDRAC', lambda m: m.reset_idrac())
-        LOG.info('Reset iDRAC for node %(node)s done',
+            redfish_utils.wait_until_get_system_ready(task.node)
-                 {'node': task.node.uuid})
+            LOG.info('Reset iDRAC for node %(node)s done',
+                     {'node': task.node.uuid})
+        except exception.RedfishError as exc:
+            if "Oem/Dell/DelliDRACCardService is missing" in str(exc):
+                LOG.warning('iDRAC on node %(node)s does not support '
+                            'iDRAC reset using the idrac-redfish driver. '
+                            'If using iDRAC9, consider upgrading firmware. '
+                            'If using iDRAC8, consider switching to '
+                            'idrac-wsman for management interface if '
+                            'possible.',
+                            {'node': task.node.uuid})
+            if task.node.provision_state != states.VERIFYING:
+                raise
 
     @METRICS.timer('DracRedfishManagement.known_good_state')
     @base.verify_step(priority=0)

ironic/tests/unit/drivers/modules/drac/test_management.py

@@ -28,6 +28,7 @@ from oslo_utils import importutils
 import ironic.common.boot_devices
 from ironic.common import exception
 from ironic.common import molds
+from ironic.common import states
 from ironic.conductor import periodics
 from ironic.conductor import task_manager
 from ironic.conductor import utils as manager_utils
@@ -1438,6 +1439,48 @@ class DracRedfishManagementTestCase(test_utils.BaseDracTest):
             mock_manager_oem.job_service.delete_jobs.assert_called_once_with(
                 job_ids=['JID_CLEARALL'])
 
+    @mock.patch.object(drac_mgmt, 'LOG', autospec=True)
+    @mock.patch.object(drac_utils, 'redfish_utils', autospec=True)
+    def test_clear_job_queue_missing_attr_verify_step(self,
+                                                      mock_redfish_utils,
+                                                      mock_log):
+        mock_system = mock_redfish_utils.get_system.return_value
+        mock_manager = mock.MagicMock()
+        mock_system.managers = [mock_manager]
+        mock_manager_oem = mock_manager.get_oem_extension.return_value
+        mock_manager_oem.job_service.delete_jobs.side_effect = (
+            exception.RedfishError("Oem/Dell/DellJobService is missing"))
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            task.node.provision_state = states.VERIFYING
+            task.driver.management.clear_job_queue(task)
+            mock_log.warning.assert_called_once_with(
+                'iDRAC on node %(node)s does not support '
+                'clearing Lifecycle Controller job queue '
+                'using the idrac-redfish driver. '
+                'If using iDRAC9, consider upgrading firmware. '
+                'If using iDRAC8, consider switching to '
+                'idrac-wsman for management interface if '
+                'possible.',
+                {'node': task.node.uuid})
+
+    @mock.patch.object(drac_mgmt, 'LOG', autospec=True)
+    @mock.patch.object(drac_utils, 'redfish_utils', autospec=True)
+    def test_clear_job_queue_missing_attr_clean_step(self,
+                                                     mock_redfish_utils,
+                                                     mock_log):
+        mock_system = mock_redfish_utils.get_system.return_value
+        mock_manager = mock.MagicMock()
+        mock_system.managers = [mock_manager]
+        mock_manager_oem = mock_manager.get_oem_extension.return_value
+        mock_manager_oem.job_service.delete_jobs.side_effect = (
+            exception.RedfishError("Oem/Dell/DellJobService is missing"))
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            task.node.provision_state = states.CLEANING
+            self.assertRaises(ironic.common.exception.RedfishError,
+                              task.driver.management.clear_job_queue, task)
+
     @mock.patch.object(redfish_utils, 'wait_until_get_system_ready',
                        autospec=True)
     @mock.patch.object(drac_utils, 'redfish_utils', autospec=True)
@@ -1452,6 +1495,53 @@ class DracRedfishManagementTestCase(test_utils.BaseDracTest):
             task.driver.management.reset_idrac(task)
             mock_manager_oem.reset_idrac.assert_called_once_with()
 
+    @mock.patch.object(redfish_utils, 'wait_until_get_system_ready',
+                       autospec=True)
+    @mock.patch.object(drac_mgmt, 'LOG', autospec=True)
+    @mock.patch.object(drac_utils, 'redfish_utils', autospec=True)
+    def test_reset_idrac_missing_attr_verify_step(self,
+                                                  mock_redfish_utils,
+                                                  mock_log,
+                                                  mock_wait_system_ready):
+        mock_system = mock_redfish_utils.get_system.return_value
+        mock_manager = mock.MagicMock()
+        mock_system.managers = [mock_manager]
+        mock_manager_oem = mock_manager.get_oem_extension.return_value
+        mock_manager_oem.reset_idrac.side_effect = (
+            exception.RedfishError("Oem/Dell/DelliDRACCardService is missing"))
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            task.node.provision_state = states.VERIFYING
+            task.driver.management.reset_idrac(task)
+            mock_log.warning.assert_called_once_with(
+                'iDRAC on node %(node)s does not support '
+                'iDRAC reset using the idrac-redfish driver. '
+                'If using iDRAC9, consider upgrading firmware. '
+                'If using iDRAC8, consider switching to '
+                'idrac-wsman for management interface if '
+                'possible.',
+                {'node': task.node.uuid})
+
+    @mock.patch.object(redfish_utils, 'wait_until_get_system_ready',
+                       autospec=True)
+    @mock.patch.object(drac_mgmt, 'LOG', autospec=True)
+    @mock.patch.object(drac_utils, 'redfish_utils', autospec=True)
+    def test_reset_idrac_missing_attr_clean_step(self,
+                                                 mock_redfish_utils,
+                                                 mock_log,
+                                                 mock_wait_system_ready):
+        mock_system = mock_redfish_utils.get_system.return_value
+        mock_manager = mock.MagicMock()
+        mock_system.managers = [mock_manager]
+        mock_manager_oem = mock_manager.get_oem_extension.return_value
+        mock_manager_oem.reset_idrac.side_effect = (
+            exception.RedfishError("Oem/Dell/DelliDRACCardService is missing"))
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            task.node.provision_state = states.CLEANING
+            self.assertRaises(ironic.common.exception.RedfishError,
+                              task.driver.management.reset_idrac, task)
+
     @mock.patch.object(redfish_utils, 'wait_until_get_system_ready',
                        autospec=True)
     @mock.patch.object(drac_utils, 'redfish_utils', autospec=True)

releasenotes/notes/skip-clear-job-queue-idrac-reset-if-attr-missing-b2a2b609c906c6c4.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    Resolved clear_job_queue and reset_idrac verify step failures which occur
+    when the functionality is not supported by the iDRAC. When this condition
+    is detected, the code in the step handles the exception and logs a warning
+    and completes successfully in case of verification steps but fails in case
+    of cleaning steps.