Allow to configure additional ipmitool retriable errors

This change allows to configure more retriable errors for ipmitool
execution that are specific to the environment it is run in.

Task: 36296
Story: 2006410
Change-Id: I4bd06ad405f87f5fb974777fc3d84e4874b4f5bb

ironic/conf/ipmi.py

@@ -49,6 +49,10 @@ opts = [
                        'that command, the default value is True. It may be '
                        'overridden by per-node \'ipmi_disable_boot_timeout\' '
                        'option in node\'s \'driver_info\' field.')),
+    cfg.MultiStrOpt('additional_retryable_ipmi_errors',
+                    default=[],
+                    help=_('Additional errors ipmitool may encounter, '
+                           'specific to the environment it is run in.')),
 ]
 
 

ironic/drivers/modules/ipmitool.py

@@ -524,8 +524,11 @@ def _exec_ipmitool(driver_info, command, check_exit_code=None,
                 return out, err
             except processutils.ProcessExecutionError as e:
                 with excutils.save_and_reraise_exception() as ctxt:
-                    err_list = [x for x in IPMITOOL_RETRYABLE_FAILURES
+                    err_list = [
-                                if x in six.text_type(e)]
+                        x for x in (
+                            IPMITOOL_RETRYABLE_FAILURES +
+                            CONF.ipmi.additional_retryable_ipmi_errors)
+                        if x in six.text_type(e)]
                     if ((time.time() > end_time)
                         or (num_tries == 0)
                         or not err_list):

ironic/tests/unit/drivers/modules/test_ipmitool.py

@@ -402,6 +402,7 @@ class IPMIToolPrivateMethodTestCaseMeta(type):
                     self, mock_exec, mock_support):
                 ipmi.LAST_CMD_TIME = {}
                 mock_support.return_value = False
+                additional_msg = "RAKP 2 HMAC is invalid"
 
                 # Return a retryable error, then an error that cannot
                 # be retried thus resulting in a single retry
@@ -410,6 +411,9 @@ class IPMIToolPrivateMethodTestCaseMeta(type):
                     processutils.ProcessExecutionError(
                         stderr=message
                     ),
+                    processutils.ProcessExecutionError(
+                        stderr="Some more info: %s" % additional_msg
+                    ),
                     processutils.ProcessExecutionError(
                         stderr="Unknown"
                     ),
@@ -420,12 +424,14 @@ class IPMIToolPrivateMethodTestCaseMeta(type):
                 # to 3 times.
                 self.config(min_command_interval=1, group='ipmi')
                 self.config(command_retry_timeout=3, group='ipmi')
+                self.config(additional_retryable_ipmi_errors=[additional_msg],
+                            group='ipmi')
 
                 self.assertRaises(processutils.ProcessExecutionError,
                                   ipmi._exec_ipmitool,
                                   self.info, 'A B C')
                 mock_support.assert_called_once_with('timing')
-                self.assertEqual(2, mock_exec.call_count)
+                self.assertEqual(3, mock_exec.call_count)
 
             return (exec_ipmitool_exception_retry,
                     exec_ipmitool_exception_retries_exceeded,

releasenotes/notes/add-configurable-ipmi-retriables-b6056f722f6ed3b0.yaml

@@ -0,0 +1,7 @@
+---
+other:
+  - |
+    This release allows to configure retryable ipmitool exceptions via
+    ``[ipmi]additional_retryable_ipmi_errors`` so that, depending on the
+    environment, operators could allow retrying ipmitool commands containing
+    specified substrings.