diff --git a/ironic_lib/disk_utils.py b/ironic_lib/disk_utils.py index 289a87ca..8ef6d0d0 100644 --- a/ironic_lib/disk_utils.py +++ b/ironic_lib/disk_utils.py @@ -401,7 +401,8 @@ def convert_image(source, dest, out_format, run_as_root=False): prlimit=_qemu_img_limits(), use_standard_locale=True) except processutils.ProcessExecutionError as e: - if ('Resource temporarily unavailable' in e.stderr + if (('Resource temporarily unavailable' in e.stderr + or 'Cannot allocate memory' in e.stderr) and attempt < CONF.disk_utils.image_convert_attempts - 1): LOG.debug('Failed to convert image, retrying. Error: %s', e) # Sync disk caches before the next attempt diff --git a/ironic_lib/tests/test_disk_utils.py b/ironic_lib/tests/test_disk_utils.py index 94fc66a7..ca167136 100644 --- a/ironic_lib/tests/test_disk_utils.py +++ b/ironic_lib/tests/test_disk_utils.py @@ -1097,6 +1097,29 @@ class OtherFunctionTestCase(base.IronicLibTestCase): convert_call, ]) + @mock.patch.object(utils, 'execute', autospec=True) + def test_convert_image_retries_alternate_error(self, execute_mock): + ret_err = 'Failed to allocate memory: Cannot allocate memory\n' + execute_mock.side_effect = [ + processutils.ProcessExecutionError(stderr=ret_err), ('', ''), + processutils.ProcessExecutionError(stderr=ret_err), ('', ''), + ('', ''), + ] + + disk_utils.convert_image('source', 'dest', 'out_format') + convert_call = mock.call('qemu-img', 'convert', '-O', + 'out_format', 'source', 'dest', + run_as_root=False, + prlimit=mock.ANY, + use_standard_locale=True) + execute_mock.assert_has_calls([ + convert_call, + mock.call('sync'), + convert_call, + mock.call('sync'), + convert_call, + ]) + @mock.patch.object(os.path, 'getsize', autospec=True) @mock.patch.object(disk_utils, 'qemu_img_info', autospec=True) def test_get_image_mb(self, mock_qinfo, mock_getsize): diff --git a/releasenotes/notes/add-additional-disk-conversion-retry-dfff93cbdf779f81.yaml b/releasenotes/notes/add-additional-disk-conversion-retry-dfff93cbdf779f81.yaml new file mode 100644 index 00000000..ff3b5c03 --- /dev/null +++ b/releasenotes/notes/add-additional-disk-conversion-retry-dfff93cbdf779f81.yaml @@ -0,0 +1,11 @@ +--- +fixes: + - | + Adds an additional error to look for in the ``qemu-img`` image conversion + retry logic to automatically retry if 'Cannot allocate memory' is + encountered, as ``qemu-img`` makes a number of memory allocation requests + and the most likely is upon creating the convesrsion thread resulting in + 'qemu: qemu_thread_create_: Resource temporarily unavailable' + but other memory allocation fails can result in + 'Failed to allocate memory: Cannot allocate memory'. Both types of errors + are now checked and automatically retried upon.