From 0b6dd9362e481dc3624e430831f463f93d09a230 Mon Sep 17 00:00:00 2001 From: Jacob Anders Date: Fri, 5 Apr 2024 08:26:17 +1000 Subject: [PATCH] Wait for BIOS configuration job to complete When reconfiguring BIOS on iDRAC nodes, there can be a conflict between Lifecycle Controller job implementing the actual BIOS change and another job implementing boot sequence override. This change resolves the issue with the exception handler by adding code dealing with HTTP 409 error which wasn't correctly handled before, causing the problem. Change-Id: Ib57a330d267f573c8481e4823b0f5ee14556fde3 --- sushy_oem_idrac/resources/manager/manager.py | 13 ++- .../unit/json_samples/error_pending_job.json | 18 ++++ .../unit/json_samples/error_running_job.json | 18 ++++ .../fix-lc-wait-loop-cebbee222d3dc7c3.yaml | 6 ++ .../unit/resources/manager/test_manager.py | 87 +++++++++++++++++++ 5 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 sushy_oem_idrac/tests/unit/json_samples/error_pending_job.json create mode 100644 sushy_oem_idrac/tests/unit/json_samples/error_running_job.json create mode 100644 sushy_oem_idrac/tests/unit/json_samples/releasenotes/notes/fix-lc-wait-loop-cebbee222d3dc7c3.yaml diff --git a/sushy_oem_idrac/resources/manager/manager.py b/sushy_oem_idrac/resources/manager/manager.py index 4c4c1a2..e6ef42c 100644 --- a/sushy_oem_idrac/resources/manager/manager.py +++ b/sushy_oem_idrac/resources/manager/manager.py @@ -238,8 +238,8 @@ VFDD\ return response - except (sushy.exceptions.ServerSideError, - sushy.exceptions.BadRequestError) as exc: + except sushy.exceptions.HTTPError as exc: + LOG.warning( 'Dell OEM set boot device failed (attempts left ' '%d): %s', attempts, exc) @@ -247,6 +247,8 @@ VFDD\ errors = exc.body and exc.body.get( '@Message.ExtendedInfo') or [] + found = False + for error in errors: message_id = error.get('MessageId') @@ -254,6 +256,7 @@ VFDD\ error.get('Message', 'Unknown error')) if constants.IDRAC_CONFIG_PENDING in message_id: + found = True if not rebooted: LOG.warning( 'Let\'s try to turn it off and on again... ' @@ -264,10 +267,14 @@ VFDD\ break elif constants.IDRAC_JOB_RUNNING in message_id: + found = True pass else: - time.sleep(self.RETRY_DELAY) + if found: + time.sleep(self.RETRY_DELAY) + else: + raise if not attempts: LOG.error('Too many (%d) retries, bailing ' diff --git a/sushy_oem_idrac/tests/unit/json_samples/error_pending_job.json b/sushy_oem_idrac/tests/unit/json_samples/error_pending_job.json new file mode 100644 index 0000000..7a61f4d --- /dev/null +++ b/sushy_oem_idrac/tests/unit/json_samples/error_pending_job.json @@ -0,0 +1,18 @@ +{ + "error": { + "@Message.ExtendedInfo": [ + { + "Message": "Unable to perform the import or export operation because there are pending attribute changes or a configuration job is in progress.", + "MessageArgs": "[]", + "MessageArgs@odata.count": 0, + "MessageId": "IDRAC.2.8.LC068", + "RelatedProperties": [], + "RelatedProperties@odata.count": 0, + "Resolution": "Apply or cancel any pending attribute changes. Changes can be applied by creating a targeted configuration job, or the changes can be cancelled by invoking the DeletePendingConfiguration method. If a configuration job is in progress, wait until it is completed before retrying the import or export system configuration operation.", + "Severity": "Warning" + } + ], + "code": "IDRAC.2.8.LC068", + "message": "Unable to perform the import or export operation because there are pending attribute changes or a configuration job is in progress." + } +} diff --git a/sushy_oem_idrac/tests/unit/json_samples/error_running_job.json b/sushy_oem_idrac/tests/unit/json_samples/error_running_job.json new file mode 100644 index 0000000..aa634a8 --- /dev/null +++ b/sushy_oem_idrac/tests/unit/json_samples/error_running_job.json @@ -0,0 +1,18 @@ +{ + "error": { + "@Message.ExtendedInfo": [ + { + "Message": "A job operation is already running. Retry the operation after the existing job is completed.", + "MessageArgs": "[]", + "MessageArgs@odata.count": 0, + "MessageId": "IDRAC.2.8.RAC0679", + "RelatedProperties": [], + "RelatedProperties@odata.count": 0, + "Resolution": "Wait until the running job is completed or delete the scheduled job and retry the operation", + "Severity": "Warning" + } + ], + "code": "IDRAC.2.8.RAC0679", + "message": "Wait until the running job is completed or delete the scheduled job and retry the operation." + } +} diff --git a/sushy_oem_idrac/tests/unit/json_samples/releasenotes/notes/fix-lc-wait-loop-cebbee222d3dc7c3.yaml b/sushy_oem_idrac/tests/unit/json_samples/releasenotes/notes/fix-lc-wait-loop-cebbee222d3dc7c3.yaml new file mode 100644 index 0000000..a9bc1a8 --- /dev/null +++ b/sushy_oem_idrac/tests/unit/json_samples/releasenotes/notes/fix-lc-wait-loop-cebbee222d3dc7c3.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Resolves the issue where wait for Lifecycle Controller job ends prematurely + due to iDRAC BMC raising a different exception type than the code + previously expected by using the correct exception class. diff --git a/sushy_oem_idrac/tests/unit/resources/manager/test_manager.py b/sushy_oem_idrac/tests/unit/resources/manager/test_manager.py index 350dff2..84fc7a9 100644 --- a/sushy_oem_idrac/tests/unit/resources/manager/test_manager.py +++ b/sushy_oem_idrac/tests/unit/resources/manager/test_manager.py @@ -14,10 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. +from http import client as http_client import json from unittest import mock from oslotest.base import BaseTestCase +import requests import sushy from sushy.resources.manager import manager from sushy.taskmonitor import TaskMonitor @@ -94,6 +96,91 @@ class ManagerTestCase(BaseTestCase): '#FirstBootDevice">VCD-DVD' ''}) + @mock.patch('time.sleep', autospec=True) + @mock.patch('sushy.resources.oem.common._global_extn_mgrs_by_resource', {}) + def test_set_virtual_boot_device_cd_running_exc(self, mock_sleep): + oem = self.manager.get_oem_extension('Dell') + + with open('sushy_oem_idrac/tests/unit/json_samples/' + 'error_running_job.json') as f: + response_obj = json.load(f) + + response1 = mock.MagicMock(spec=requests.Response) + response1.status_code = http_client.BAD_REQUEST + response1.json.return_value = response_obj + response1.code = "IDRAC.2.8.LC068" + + response2 = mock.MagicMock(spec=requests.Response) + response2.status_code = http_client.OK + + self.conn.post.side_effect = [sushy.exceptions.HTTPError( + method='POST', url=self.manager.path, response=response1), + response2] + + oem.set_virtual_boot_device( + sushy.VIRTUAL_MEDIA_CD, manager=self.manager) + + self.conn.post.assert_called_with( + '/redfish/v1/Managers/iDRAC.Embedded.1/Actions/Oem/EID_674_Manager' + '.ImportSystemConfiguration', + data={'ShareParameters': {'Target': 'ALL'}, + 'ImportBuffer': + '' + 'Enabled' + 'VCD-DVD' + ''}) + + @mock.patch('sushy_oem_idrac.utils.reboot_system', autospec=True) + @mock.patch('sushy.resources.oem.common._global_extn_mgrs_by_resource', {}) + def test_set_virtual_boot_device_cd_pending_exc(self, mock_reboot): + oem = self.manager.get_oem_extension('Dell') + + with open('sushy_oem_idrac/tests/unit/json_samples/' + 'error_pending_job.json') as f: + response_obj = json.load(f) + + response1 = mock.MagicMock(spec=requests.Response) + response1.status_code = http_client.BAD_REQUEST + response1.json.return_value = response_obj + response1.code = "IDRAC.2.8.LC068" + + response2 = mock.MagicMock(spec=requests.Response) + response2.status_code = http_client.OK + + self.conn.post.side_effect = [sushy.exceptions.HTTPError( + method='POST', url=self.manager.path, response=response1), + response2] + + oem.set_virtual_boot_device( + sushy.VIRTUAL_MEDIA_CD, manager=self.manager) + + self.conn.post.assert_called_with( + '/redfish/v1/Managers/iDRAC.Embedded.1/Actions/Oem/EID_674_Manager' + '.ImportSystemConfiguration', + data={'ShareParameters': {'Target': 'ALL'}, + 'ImportBuffer': + '' + 'Enabled' + 'VCD-DVD' + ''}) + + @mock.patch('sushy.resources.oem.common._global_extn_mgrs_by_resource', {}) + def test_set_virtual_boot_device_cd_other_exc(self): + oem = self.manager.get_oem_extension('Dell') + + response = mock.MagicMock(spec=requests.Response) + response.status_code = http_client.FORBIDDEN + + self.conn.post.side_effect = sushy.exceptions.HTTPError( + method='POST', url=self.manager.path, response=response) + + self.assertRaises(sushy.exceptions.HTTPError, + oem.set_virtual_boot_device, + sushy.VIRTUAL_MEDIA_CD, + manager=self.manager) + @mock.patch('sushy.resources.oem.common._global_extn_mgrs_by_resource', {}) def test_get_allowed_export_target_values(self): oem = self.manager.get_oem_extension('Dell')