From 000ea74d3407d13d28e4aaef99b82c1c2563bf12 Mon Sep 17 00:00:00 2001
From: Julia Kreger <juliaashleykreger@gmail.com>
Date: Tue, 8 Dec 2020 14:02:04 -0800
Subject: [PATCH] Fix other out of memory error for qemu-img convert

qemu-img convert makes a number of allocation requests, and less
common than the "Resource temporarily unavailable" error is
"Cannot allocate memory". The first seems to be largely raised
upon a thread creation for the image conversion, where the first
appears to be on buffer setup on the original process.

Regardless, we should check for both and retry conversion
automatically.

rhbz# 1892773

Note: Because of a later patch which transformed the retry logic to
leverage tenacity, this change has been modified accordingly.

Change-Id: If435ed0efe034d4bb99f0281ebc73840c0f38fb3
(cherry picked from commit 9be6afc751f0dbc251c4b5eeed71fd3b3659976c)
(cherry picked from commit 9790280301ae2903906f2a583ccadec270029793)
---
 ironic_lib/disk_utils.py                      |  3 ++-
 ironic_lib/tests/test_disk_utils.py           | 23 +++++++++++++++++++
 ...isk-conversion-retry-dfff93cbdf779f81.yaml | 11 +++++++++
 3 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 releasenotes/notes/add-additional-disk-conversion-retry-dfff93cbdf779f81.yaml

diff --git a/ironic_lib/disk_utils.py b/ironic_lib/disk_utils.py
index 289a87ca..8ef6d0d0 100644
--- a/ironic_lib/disk_utils.py
+++ b/ironic_lib/disk_utils.py
@@ -401,7 +401,8 @@ def convert_image(source, dest, out_format, run_as_root=False):
                           prlimit=_qemu_img_limits(),
                           use_standard_locale=True)
         except processutils.ProcessExecutionError as e:
-            if ('Resource temporarily unavailable' in e.stderr
+            if (('Resource temporarily unavailable' in e.stderr
+                    or 'Cannot allocate memory' in e.stderr)
                     and attempt < CONF.disk_utils.image_convert_attempts - 1):
                 LOG.debug('Failed to convert image, retrying. Error: %s', e)
                 # Sync disk caches before the next attempt
diff --git a/ironic_lib/tests/test_disk_utils.py b/ironic_lib/tests/test_disk_utils.py
index 94fc66a7..ca167136 100644
--- a/ironic_lib/tests/test_disk_utils.py
+++ b/ironic_lib/tests/test_disk_utils.py
@@ -1097,6 +1097,29 @@ class OtherFunctionTestCase(base.IronicLibTestCase):
             convert_call,
         ])
 
+    @mock.patch.object(utils, 'execute', autospec=True)
+    def test_convert_image_retries_alternate_error(self, execute_mock):
+        ret_err = 'Failed to allocate memory: Cannot allocate memory\n'
+        execute_mock.side_effect = [
+            processutils.ProcessExecutionError(stderr=ret_err), ('', ''),
+            processutils.ProcessExecutionError(stderr=ret_err), ('', ''),
+            ('', ''),
+        ]
+
+        disk_utils.convert_image('source', 'dest', 'out_format')
+        convert_call = mock.call('qemu-img', 'convert', '-O',
+                                 'out_format', 'source', 'dest',
+                                 run_as_root=False,
+                                 prlimit=mock.ANY,
+                                 use_standard_locale=True)
+        execute_mock.assert_has_calls([
+            convert_call,
+            mock.call('sync'),
+            convert_call,
+            mock.call('sync'),
+            convert_call,
+        ])
+
     @mock.patch.object(os.path, 'getsize', autospec=True)
     @mock.patch.object(disk_utils, 'qemu_img_info', autospec=True)
     def test_get_image_mb(self, mock_qinfo, mock_getsize):
diff --git a/releasenotes/notes/add-additional-disk-conversion-retry-dfff93cbdf779f81.yaml b/releasenotes/notes/add-additional-disk-conversion-retry-dfff93cbdf779f81.yaml
new file mode 100644
index 00000000..ff3b5c03
--- /dev/null
+++ b/releasenotes/notes/add-additional-disk-conversion-retry-dfff93cbdf779f81.yaml
@@ -0,0 +1,11 @@
+---
+fixes:
+  - |
+    Adds an additional error to look for in the ``qemu-img`` image conversion
+    retry logic to automatically retry if 'Cannot allocate memory' is
+    encountered, as ``qemu-img`` makes a number of memory allocation requests
+    and the most likely is upon creating the convesrsion thread resulting in
+    'qemu: qemu_thread_create_: Resource temporarily unavailable'
+    but other memory allocation fails can result in
+    'Failed to allocate memory: Cannot allocate memory'. Both types of errors
+    are now checked and automatically retried upon.