Fix other out of memory error for qemu-img convert

qemu-img convert makes a number of allocation requests, and less
common than the "Resource temporarily unavailable" error is
"Cannot allocate memory". The first seems to be largely raised
upon a thread creation for the image conversion, where the first
appears to be on buffer setup on the original process.

Regardless, we should check for both and retry conversion
automatically.

rhbz# 1892773

Change-Id: If435ed0efe034d4bb99f0281ebc73840c0f38fb3
(cherry picked from commit 9be6afc751)
This commit is contained in:
Julia Kreger 2020-12-08 14:02:04 -08:00
parent e1b5e12cb1
commit 9790280301
3 changed files with 38 additions and 2 deletions

View File

@ -394,7 +394,8 @@ def qemu_img_info(path):
def _retry_on_res_temp_unavailable(exc): def _retry_on_res_temp_unavailable(exc):
if (isinstance(exc, processutils.ProcessExecutionError) if (isinstance(exc, processutils.ProcessExecutionError)
and 'Resource temporarily unavailable' in exc.stderr): and ('Resource temporarily unavailable' in exc.stderr
or 'Cannot allocate memory' in exc.stderr)):
return True return True
return False return False
@ -411,7 +412,8 @@ def convert_image(source, dest, out_format, run_as_root=False):
prlimit=_qemu_img_limits(), prlimit=_qemu_img_limits(),
use_standard_locale=True) use_standard_locale=True)
except processutils.ProcessExecutionError as e: except processutils.ProcessExecutionError as e:
if 'Resource temporarily unavailable' in e.stderr: if ('Resource temporarily unavailable' in e.stderr
or 'Cannot allocate memory' in e.stderr):
LOG.debug('Failed to convert image, retrying. Error: %s', e) LOG.debug('Failed to convert image, retrying. Error: %s', e)
# Sync disk caches before the next attempt # Sync disk caches before the next attempt
utils.execute('sync') utils.execute('sync')

View File

@ -1068,6 +1068,29 @@ class OtherFunctionTestCase(base.IronicLibTestCase):
convert_call, convert_call,
]) ])
@mock.patch.object(utils, 'execute', autospec=True)
def test_convert_image_retries_alternate_error(self, execute_mock):
ret_err = 'Failed to allocate memory: Cannot allocate memory\n'
execute_mock.side_effect = [
processutils.ProcessExecutionError(stderr=ret_err), ('', ''),
processutils.ProcessExecutionError(stderr=ret_err), ('', ''),
('', ''),
]
disk_utils.convert_image('source', 'dest', 'out_format')
convert_call = mock.call('qemu-img', 'convert', '-O',
'out_format', 'source', 'dest',
run_as_root=False,
prlimit=mock.ANY,
use_standard_locale=True)
execute_mock.assert_has_calls([
convert_call,
mock.call('sync'),
convert_call,
mock.call('sync'),
convert_call,
])
@mock.patch.object(utils, 'execute', autospec=True) @mock.patch.object(utils, 'execute', autospec=True)
def test_convert_image_retries_and_fails(self, execute_mock): def test_convert_image_retries_and_fails(self, execute_mock):
ret_err = 'qemu: qemu_thread_create: Resource temporarily unavailable' ret_err = 'qemu: qemu_thread_create: Resource temporarily unavailable'

View File

@ -0,0 +1,11 @@
---
fixes:
- |
Adds an additional error to look for in the ``qemu-img`` image conversion
retry logic to automatically retry if 'Cannot allocate memory' is
encountered, as ``qemu-img`` makes a number of memory allocation requests
and the most likely is upon creating the convesrsion thread resulting in
'qemu: qemu_thread_create_: Resource temporarily unavailable'
but other memory allocation fails can result in
'Failed to allocate memory: Cannot allocate memory'. Both types of errors
are now checked and automatically retried upon.