Merge "Fix other out of memory error for qemu-img convert" into stable/victoria

ironic_lib/disk_utils.py

@@ -394,7 +394,8 @@ def qemu_img_info(path):
 
 def _retry_on_res_temp_unavailable(exc):
     if (isinstance(exc, processutils.ProcessExecutionError)
-            and 'Resource temporarily unavailable' in exc.stderr):
+            and ('Resource temporarily unavailable' in exc.stderr
+                 or 'Cannot allocate memory' in exc.stderr)):
         return True
     return False
 
@@ -411,7 +412,8 @@ def convert_image(source, dest, out_format, run_as_root=False):
                       prlimit=_qemu_img_limits(),
                       use_standard_locale=True)
     except processutils.ProcessExecutionError as e:
-        if 'Resource temporarily unavailable' in e.stderr:
+        if ('Resource temporarily unavailable' in e.stderr
+            or 'Cannot allocate memory' in e.stderr):
             LOG.debug('Failed to convert image, retrying. Error: %s', e)
             # Sync disk caches before the next attempt
             utils.execute('sync')

ironic_lib/tests/test_disk_utils.py

@@ -1068,6 +1068,29 @@ class OtherFunctionTestCase(base.IronicLibTestCase):
             convert_call,
         ])
 
+    @mock.patch.object(utils, 'execute', autospec=True)
+    def test_convert_image_retries_alternate_error(self, execute_mock):
+        ret_err = 'Failed to allocate memory: Cannot allocate memory\n'
+        execute_mock.side_effect = [
+            processutils.ProcessExecutionError(stderr=ret_err), ('', ''),
+            processutils.ProcessExecutionError(stderr=ret_err), ('', ''),
+            ('', ''),
+        ]
+
+        disk_utils.convert_image('source', 'dest', 'out_format')
+        convert_call = mock.call('qemu-img', 'convert', '-O',
+                                 'out_format', 'source', 'dest',
+                                 run_as_root=False,
+                                 prlimit=mock.ANY,
+                                 use_standard_locale=True)
+        execute_mock.assert_has_calls([
+            convert_call,
+            mock.call('sync'),
+            convert_call,
+            mock.call('sync'),
+            convert_call,
+        ])
+
     @mock.patch.object(utils, 'execute', autospec=True)
     def test_convert_image_retries_and_fails(self, execute_mock):
         ret_err = 'qemu: qemu_thread_create: Resource temporarily unavailable'

releasenotes/notes/add-additional-disk-conversion-retry-dfff93cbdf779f81.yaml

@@ -0,0 +1,11 @@
+---
+fixes:
+  - |
+    Adds an additional error to look for in the ``qemu-img`` image conversion
+    retry logic to automatically retry if 'Cannot allocate memory' is
+    encountered, as ``qemu-img`` makes a number of memory allocation requests
+    and the most likely is upon creating the convesrsion thread resulting in
+    'qemu: qemu_thread_create_: Resource temporarily unavailable'
+    but other memory allocation fails can result in
+    'Failed to allocate memory: Cannot allocate memory'. Both types of errors
+    are now checked and automatically retried upon.