Merge "Fix redfish RAID failed tasks" into stable/xena
This commit is contained in:
commit
443fd8be85
|
@ -1113,60 +1113,55 @@ class RedfishRAID(base.RAIDInterface):
|
||||||
'%(node)s was already locked by another process. '
|
'%(node)s was already locked by another process. '
|
||||||
'Skip.', {'node': node_uuid})
|
'Skip.', {'node': node_uuid})
|
||||||
|
|
||||||
def _get_error_messages(self, response):
|
|
||||||
try:
|
|
||||||
body = response.json()
|
|
||||||
except ValueError:
|
|
||||||
return []
|
|
||||||
else:
|
|
||||||
error = body.get('error', {})
|
|
||||||
code = error.get('code', '')
|
|
||||||
message = error.get('message', code)
|
|
||||||
ext_info = error.get('@Message.ExtendedInfo', [{}])
|
|
||||||
messages = [m.get('Message') for m in ext_info if 'Message' in m]
|
|
||||||
if not messages and message:
|
|
||||||
messages = [message]
|
|
||||||
return messages
|
|
||||||
|
|
||||||
def _raid_config_in_progress(self, task, raid_config):
|
def _raid_config_in_progress(self, task, raid_config):
|
||||||
"""Check if this RAID configuration operation is still in progress."""
|
"""Check if this RAID configuration operation is still in progress.
|
||||||
|
|
||||||
|
:param task: TaskManager object containing the node.
|
||||||
|
:param raid_config: RAID configuration operation details.
|
||||||
|
:returns: True, if still in progress, otherwise False.
|
||||||
|
"""
|
||||||
task_monitor_uri = raid_config['task_monitor_uri']
|
task_monitor_uri = raid_config['task_monitor_uri']
|
||||||
try:
|
try:
|
||||||
task_monitor = redfish_utils.get_task_monitor(task.node,
|
task_monitor = redfish_utils.get_task_monitor(task.node,
|
||||||
task_monitor_uri)
|
task_monitor_uri)
|
||||||
except exception.RedfishError:
|
except exception.RedfishError:
|
||||||
LOG.info('Unable to get status of RAID %(operation)s task to node '
|
LOG.info('Unable to get status of RAID %(operation)s task '
|
||||||
'%(node_uuid)s; assuming task completed successfully',
|
'%(task_mon_uri)s to node %(node_uuid)s; assuming task '
|
||||||
|
'completed successfully',
|
||||||
{'operation': raid_config['operation'],
|
{'operation': raid_config['operation'],
|
||||||
|
'task_mon_uri': task_monitor_uri,
|
||||||
'node_uuid': task.node.uuid})
|
'node_uuid': task.node.uuid})
|
||||||
return False
|
return False
|
||||||
if task_monitor.is_processing:
|
if task_monitor.is_processing:
|
||||||
LOG.debug('RAID %(operation)s task %(task_mon)s to node '
|
LOG.debug('RAID %(operation)s task %(task_mon_uri)s to node '
|
||||||
'%(node_uuid)s still in progress',
|
'%(node_uuid)s still in progress',
|
||||||
{'operation': raid_config['operation'],
|
{'operation': raid_config['operation'],
|
||||||
'task_mon': task_monitor.task_monitor_uri,
|
'task_mon_uri': task_monitor.task_monitor_uri,
|
||||||
'node_uuid': task.node.uuid})
|
'node_uuid': task.node.uuid})
|
||||||
return True
|
return True
|
||||||
else:
|
else:
|
||||||
response = task_monitor.response
|
sushy_task = task_monitor.get_task()
|
||||||
if response is not None:
|
messages = []
|
||||||
status_code = response.status_code
|
if sushy_task.messages and not sushy_task.messages[0].message:
|
||||||
if status_code >= 400:
|
sushy_task.parse_messages()
|
||||||
messages = self._get_error_messages(response)
|
|
||||||
LOG.error('RAID %(operation)s task to node '
|
messages = [m.message for m in sushy_task.messages]
|
||||||
'%(node_uuid)s failed with status '
|
|
||||||
'%(status_code)s; messages: %(messages)s',
|
if (sushy_task.task_state == sushy.TASK_STATE_COMPLETED
|
||||||
{'operation': raid_config['operation'],
|
and sushy_task.task_status in
|
||||||
'node_uuid': task.node.uuid,
|
[sushy.HEALTH_OK, sushy.HEALTH_WARNING]):
|
||||||
'status_code': status_code,
|
LOG.info('RAID %(operation)s task %(task_mon_uri)s to node '
|
||||||
'messages': ", ".join(messages)})
|
'%(node_uuid)s completed.',
|
||||||
else:
|
{'operation': raid_config['operation'],
|
||||||
LOG.info('RAID %(operation)s task to node '
|
'task_mon_uri': task_monitor.task_monitor_uri,
|
||||||
'%(node_uuid)s completed with status '
|
'node_uuid': task.node.uuid})
|
||||||
'%(status_code)s',
|
else:
|
||||||
{'operation': raid_config['operation'],
|
LOG.error('RAID %(operation)s task %(task_mon_uri)s to node '
|
||||||
'node_uuid': task.node.uuid,
|
'%(node_uuid)s failed; messages: %(messages)s',
|
||||||
'status_code': status_code})
|
{'operation': raid_config['operation'],
|
||||||
|
'task_mon_uri': task_monitor.task_monitor_uri,
|
||||||
|
'node_uuid': task.node.uuid,
|
||||||
|
'messages': ", ".join(messages)})
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@METRICS.timer('RedfishRAID._check_node_raid_config')
|
@METRICS.timer('RedfishRAID._check_node_raid_config')
|
||||||
|
|
|
@ -1058,3 +1058,87 @@ class RedfishRAIDTestCase(db_base.DbTestCase):
|
||||||
|
|
||||||
self.assertEqual(storage, self.mock_storage)
|
self.assertEqual(storage, self.mock_storage)
|
||||||
nonraid_storage.drives.assert_not_called()
|
nonraid_storage.drives.assert_not_called()
|
||||||
|
|
||||||
|
@mock.patch.object(redfish_utils, 'get_task_monitor', autospec=True)
|
||||||
|
@mock.patch.object(redfish_raid.LOG, 'info', autospec=True)
|
||||||
|
def test__raid_config_in_progress_success(
|
||||||
|
self, mock_info, mock_get_task_monitor, mock_get_system):
|
||||||
|
mock_task = mock.Mock()
|
||||||
|
mock_task.task_state = sushy.TASK_STATE_COMPLETED
|
||||||
|
mock_task.task_status = sushy.HEALTH_OK
|
||||||
|
mock_task.messages = []
|
||||||
|
mock_task_monitor = mock.Mock()
|
||||||
|
mock_task_monitor.is_processing = False
|
||||||
|
mock_task_monitor.get_task.return_value = mock_task
|
||||||
|
mock_get_task_monitor.return_value = mock_task_monitor
|
||||||
|
|
||||||
|
with task_manager.acquire(self.context, self.node.uuid,
|
||||||
|
shared=False) as task:
|
||||||
|
|
||||||
|
raid = redfish_raid.RedfishRAID()
|
||||||
|
result = raid._raid_config_in_progress(
|
||||||
|
task, {'task_monitor_uri': '/TaskService/123',
|
||||||
|
'operation': 'create'})
|
||||||
|
self.assertEqual(False, result)
|
||||||
|
mock_info.assert_called_once()
|
||||||
|
|
||||||
|
@mock.patch.object(redfish_utils, 'get_task_monitor', autospec=True)
|
||||||
|
@mock.patch.object(redfish_raid.LOG, 'info', autospec=True)
|
||||||
|
def test__raid_config_in_progress_task_mon_error(
|
||||||
|
self, mock_info, mock_get_task_monitor, mock_get_system):
|
||||||
|
mock_get_task_monitor.side_effect = exception.RedfishError(
|
||||||
|
error='Task not found')
|
||||||
|
|
||||||
|
with task_manager.acquire(self.context, self.node.uuid,
|
||||||
|
shared=False) as task:
|
||||||
|
|
||||||
|
raid = redfish_raid.RedfishRAID()
|
||||||
|
result = raid._raid_config_in_progress(
|
||||||
|
task, {'task_monitor_uri': '/TaskService/123',
|
||||||
|
'operation': 'create'})
|
||||||
|
self.assertEqual(False, result)
|
||||||
|
mock_info.assert_called_once()
|
||||||
|
|
||||||
|
@mock.patch.object(redfish_utils, 'get_task_monitor', autospec=True)
|
||||||
|
@mock.patch.object(redfish_raid.LOG, 'debug', autospec=True)
|
||||||
|
def test__raid_config_in_progress_still_processing(
|
||||||
|
self, mock_debug, mock_get_task_monitor, mock_get_system):
|
||||||
|
mock_task_monitor = mock.Mock()
|
||||||
|
mock_task_monitor.is_processing = True
|
||||||
|
mock_get_task_monitor.return_value = mock_task_monitor
|
||||||
|
|
||||||
|
with task_manager.acquire(self.context, self.node.uuid,
|
||||||
|
shared=False) as task:
|
||||||
|
|
||||||
|
raid = redfish_raid.RedfishRAID()
|
||||||
|
result = raid._raid_config_in_progress(
|
||||||
|
task, {'task_monitor_uri': '/TaskService/123',
|
||||||
|
'operation': 'create'})
|
||||||
|
self.assertEqual(True, result)
|
||||||
|
mock_debug.assert_called_once()
|
||||||
|
|
||||||
|
@mock.patch.object(redfish_utils, 'get_task_monitor', autospec=True)
|
||||||
|
@mock.patch.object(redfish_raid.LOG, 'error', autospec=True)
|
||||||
|
def test__raid_config_in_progress_failed(
|
||||||
|
self, mock_error, mock_get_task_monitor, mock_get_system):
|
||||||
|
mock_message = mock.Mock()
|
||||||
|
mock_message.message = 'RAID configuration failed'
|
||||||
|
mock_message.severity = sushy.SEVERITY_CRITICAL
|
||||||
|
mock_task = mock.Mock()
|
||||||
|
mock_task.task_state = sushy.TASK_STATE_COMPLETED
|
||||||
|
mock_task.task_status = sushy.HEALTH_CRITICAL
|
||||||
|
mock_task.messages = [mock_message]
|
||||||
|
mock_task_monitor = mock.Mock()
|
||||||
|
mock_task_monitor.is_processing = False
|
||||||
|
mock_task_monitor.get_task.return_value = mock_task
|
||||||
|
mock_get_task_monitor.return_value = mock_task_monitor
|
||||||
|
|
||||||
|
with task_manager.acquire(self.context, self.node.uuid,
|
||||||
|
shared=False) as task:
|
||||||
|
|
||||||
|
raid = redfish_raid.RedfishRAID()
|
||||||
|
result = raid._raid_config_in_progress(
|
||||||
|
task, {'task_monitor_uri': '/TaskService/123',
|
||||||
|
'operation': 'create'})
|
||||||
|
self.assertEqual(False, result)
|
||||||
|
mock_error.assert_called_once()
|
||||||
|
|
|
@ -173,6 +173,7 @@ SUSHY_SPEC = (
|
||||||
'TASK_STATE_COMPLETED',
|
'TASK_STATE_COMPLETED',
|
||||||
'HEALTH_OK',
|
'HEALTH_OK',
|
||||||
'HEALTH_WARNING',
|
'HEALTH_WARNING',
|
||||||
|
'HEALTH_CRITICAL',
|
||||||
'SECURE_BOOT_RESET_KEYS_TO_DEFAULT',
|
'SECURE_BOOT_RESET_KEYS_TO_DEFAULT',
|
||||||
'SECURE_BOOT_RESET_KEYS_DELETE_ALL',
|
'SECURE_BOOT_RESET_KEYS_DELETE_ALL',
|
||||||
'VOLUME_TYPE_RAW_DEVICE',
|
'VOLUME_TYPE_RAW_DEVICE',
|
||||||
|
|
|
@ -241,6 +241,7 @@ if not sushy:
|
||||||
TASK_STATE_COMPLETED='completed',
|
TASK_STATE_COMPLETED='completed',
|
||||||
HEALTH_OK='ok',
|
HEALTH_OK='ok',
|
||||||
HEALTH_WARNING='warning',
|
HEALTH_WARNING='warning',
|
||||||
|
HEALTH_CRITICAL='critical',
|
||||||
SECURE_BOOT_RESET_KEYS_TO_DEFAULT="ResetAllKeysToDefault",
|
SECURE_BOOT_RESET_KEYS_TO_DEFAULT="ResetAllKeysToDefault",
|
||||||
SECURE_BOOT_RESET_KEYS_DELETE_ALL="DeleteAllKeys",
|
SECURE_BOOT_RESET_KEYS_DELETE_ALL="DeleteAllKeys",
|
||||||
VOLUME_TYPE_RAW_DEVICE='rawdevice',
|
VOLUME_TYPE_RAW_DEVICE='rawdevice',
|
||||||
|
|
|
@ -0,0 +1,6 @@
|
||||||
|
---
|
||||||
|
fixes:
|
||||||
|
- |
|
||||||
|
Fixes the determination of a failed RAID configuration task in the
|
||||||
|
``redfish`` hardware type. Prior to this fix the tasks that have failed
|
||||||
|
were reported as successful.
|
Loading…
Reference in New Issue