diff --git a/ironic_lib/disk_utils.py b/ironic_lib/disk_utils.py index 30fbe0f3..a31390bb 100644 --- a/ironic_lib/disk_utils.py +++ b/ironic_lib/disk_utils.py @@ -394,7 +394,8 @@ def qemu_img_info(path): def _retry_on_res_temp_unavailable(exc): if (isinstance(exc, processutils.ProcessExecutionError) - and 'Resource temporarily unavailable' in exc.stderr): + and ('Resource temporarily unavailable' in exc.stderr + or 'Cannot allocate memory' in exc.stderr)): return True return False @@ -411,7 +412,8 @@ def convert_image(source, dest, out_format, run_as_root=False): prlimit=_qemu_img_limits(), use_standard_locale=True) except processutils.ProcessExecutionError as e: - if 'Resource temporarily unavailable' in e.stderr: + if ('Resource temporarily unavailable' in e.stderr + or 'Cannot allocate memory' in e.stderr): LOG.debug('Failed to convert image, retrying. Error: %s', e) # Sync disk caches before the next attempt utils.execute('sync') diff --git a/ironic_lib/tests/test_disk_utils.py b/ironic_lib/tests/test_disk_utils.py index b7a4de84..d163c8e2 100644 --- a/ironic_lib/tests/test_disk_utils.py +++ b/ironic_lib/tests/test_disk_utils.py @@ -1068,6 +1068,29 @@ class OtherFunctionTestCase(base.IronicLibTestCase): convert_call, ]) + @mock.patch.object(utils, 'execute', autospec=True) + def test_convert_image_retries_alternate_error(self, execute_mock): + ret_err = 'Failed to allocate memory: Cannot allocate memory\n' + execute_mock.side_effect = [ + processutils.ProcessExecutionError(stderr=ret_err), ('', ''), + processutils.ProcessExecutionError(stderr=ret_err), ('', ''), + ('', ''), + ] + + disk_utils.convert_image('source', 'dest', 'out_format') + convert_call = mock.call('qemu-img', 'convert', '-O', + 'out_format', 'source', 'dest', + run_as_root=False, + prlimit=mock.ANY, + use_standard_locale=True) + execute_mock.assert_has_calls([ + convert_call, + mock.call('sync'), + convert_call, + mock.call('sync'), + convert_call, + ]) + @mock.patch.object(utils, 'execute', autospec=True) def test_convert_image_retries_and_fails(self, execute_mock): ret_err = 'qemu: qemu_thread_create: Resource temporarily unavailable' diff --git a/releasenotes/notes/add-additional-disk-conversion-retry-dfff93cbdf779f81.yaml b/releasenotes/notes/add-additional-disk-conversion-retry-dfff93cbdf779f81.yaml new file mode 100644 index 00000000..ff3b5c03 --- /dev/null +++ b/releasenotes/notes/add-additional-disk-conversion-retry-dfff93cbdf779f81.yaml @@ -0,0 +1,11 @@ +--- +fixes: + - | + Adds an additional error to look for in the ``qemu-img`` image conversion + retry logic to automatically retry if 'Cannot allocate memory' is + encountered, as ``qemu-img`` makes a number of memory allocation requests + and the most likely is upon creating the convesrsion thread resulting in + 'qemu: qemu_thread_create_: Resource temporarily unavailable' + but other memory allocation fails can result in + 'Failed to allocate memory: Cannot allocate memory'. Both types of errors + are now checked and automatically retried upon.