Merge "Retry on iDRAC SYS518 errors for all requests"

releasenotes/notes/fix-retry_volume_operation-on_sys518-009f2b16e5c38a27.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    Add retries on iDRAC error with code SYS518 and message "iDRAC is
+    currently unable to display any information because data sources are
+    unavailable." for all request types in addition to existing GET methods.
+    This helps to fix a known intermittent issue when deleting set of volumes
+    one after another and iDRAC is reloading after deleting each volume.

sushy/connector.py

@@ -85,6 +85,15 @@ class Connector(object):
         """Close this connector and the associated HTTP session."""
         self._session.close()
 
+    def check_retry_on_exception(self, exception_msg):
+        """Checks whether retry on exception is required."""
+        if ('SYS518' in str(exception_msg)):
+            LOG.debug('iDRAC is not yet ready after previous operation. '
+                      'Error: %(err)s', {'err': str(exception_msg)})
+            return True
+        else:
+            return False
+
     def _op(self, method, path='', data=None, headers=None, blocking=False,
             timeout=60, **extra_session_req_kwargs):
         """Generic RESTful request handler.
@@ -197,9 +206,9 @@ class Connector(object):
                               "%s", e.message)
                 raise
         except exceptions.ServerSideError as e:
-            if method.lower() != 'get' or self._server_side_retries <= 0:
-                raise
-            else:
+            if ((method.lower() == 'get'
+                or self.check_retry_on_exception(e.message))
+                    and self._server_side_retries > 0):
                 LOG.warning('Got server side error %s in response to a '
                             'GET request, retrying after %d seconds',
                             e, self._server_side_retries)
@@ -208,6 +217,8 @@ class Connector(object):
                 return self._op(method, path, data=data, headers=headers,
                                 blocking=blocking, timeout=timeout,
                                 **extra_session_req_kwargs)
+            else:
+                raise
 
         if blocking and response.status_code == 202:
             if not response.headers.get('Location'):

sushy/tests/unit/test_connector.py

@@ -408,6 +408,42 @@ class ConnectorOpTestCase(base.TestCase):
         self.assertEqual(10, mock_sleep.call_count)
         self.assertEqual(11, self.request.call_count)
 
+    @mock.patch('time.sleep', autospec=True)
+    def test_op_retry_on_server_500_sys518(self, mock_sleep):
+        response_info = {"error": {"@Message.ExtendedInfo": [
+            {'MessageId': 'IDRAC.2.7.SYS518'}]}}
+        mock_error = mock.Mock()
+        mock_error.status_code = 500
+        mock_error.json.return_value = response_info
+        self.request.return_value.status_code = (
+            http_client.INTERNAL_SERVER_ERROR)
+        self.request.return_value.json.side_effect =\
+            exceptions.ServerSideError(
+                method='DELETE', url='http://foo.bar', response=mock_error)
+
+        self.assertRaises(exceptions.ServerSideError, self.conn._op, 'DELETE',
+                          'http://foo.bar')
+        self.assertEqual(10, mock_sleep.call_count)
+        self.assertEqual(11, self.request.call_count)
+
+    @mock.patch('time.sleep', autospec=True)
+    def test_op_retry_on_server_500_other_than_sys518(self, mock_sleep):
+        response_info = {"error": {"@Message.ExtendedInfo": [
+            {'MessageId': 'IDRAC.2.7.SYS999'}]}}
+        mock_error = mock.Mock()
+        mock_error.status_code = 500
+        mock_error.json.return_value = response_info
+        self.request.return_value.status_code = (
+            http_client.INTERNAL_SERVER_ERROR)
+        self.request.return_value.json.side_effect =\
+            exceptions.ServerSideError(
+                method='DELETE', url='http://foo.bar', response=mock_error)
+
+        self.assertRaises(exceptions.ServerSideError, self.conn._op, 'DELETE',
+                          'http://foo.bar')
+        self.assertEqual(0, mock_sleep.call_count)
+        self.assertEqual(1, self.request.call_count)
+
     def test_access_error(self):
         self.conn._auth = None