Merge "Fix other out of memory error for qemu-img convert"

This commit is contained in:
Zuul 2020-12-14 09:04:49 +00:00 committed by Gerrit Code Review
commit 3702e26c85
3 changed files with 38 additions and 2 deletions

View File

@ -394,7 +394,8 @@ def qemu_img_info(path):
def _retry_on_res_temp_unavailable(exc):
if (isinstance(exc, processutils.ProcessExecutionError)
and 'Resource temporarily unavailable' in exc.stderr):
and ('Resource temporarily unavailable' in exc.stderr
or 'Cannot allocate memory' in exc.stderr)):
return True
return False
@ -411,7 +412,8 @@ def convert_image(source, dest, out_format, run_as_root=False):
prlimit=_qemu_img_limits(),
use_standard_locale=True)
except processutils.ProcessExecutionError as e:
if 'Resource temporarily unavailable' in e.stderr:
if ('Resource temporarily unavailable' in e.stderr
or 'Cannot allocate memory' in e.stderr):
LOG.debug('Failed to convert image, retrying. Error: %s', e)
# Sync disk caches before the next attempt
utils.execute('sync')

View File

@ -1068,6 +1068,29 @@ class OtherFunctionTestCase(base.IronicLibTestCase):
convert_call,
])
@mock.patch.object(utils, 'execute', autospec=True)
def test_convert_image_retries_alternate_error(self, execute_mock):
ret_err = 'Failed to allocate memory: Cannot allocate memory\n'
execute_mock.side_effect = [
processutils.ProcessExecutionError(stderr=ret_err), ('', ''),
processutils.ProcessExecutionError(stderr=ret_err), ('', ''),
('', ''),
]
disk_utils.convert_image('source', 'dest', 'out_format')
convert_call = mock.call('qemu-img', 'convert', '-O',
'out_format', 'source', 'dest',
run_as_root=False,
prlimit=mock.ANY,
use_standard_locale=True)
execute_mock.assert_has_calls([
convert_call,
mock.call('sync'),
convert_call,
mock.call('sync'),
convert_call,
])
@mock.patch.object(utils, 'execute', autospec=True)
def test_convert_image_retries_and_fails(self, execute_mock):
ret_err = 'qemu: qemu_thread_create: Resource temporarily unavailable'

View File

@ -0,0 +1,11 @@
---
fixes:
- |
Adds an additional error to look for in the ``qemu-img`` image conversion
retry logic to automatically retry if 'Cannot allocate memory' is
encountered, as ``qemu-img`` makes a number of memory allocation requests
and the most likely is upon creating the convesrsion thread resulting in
'qemu: qemu_thread_create_: Resource temporarily unavailable'
but other memory allocation fails can result in
'Failed to allocate memory: Cannot allocate memory'. Both types of errors
are now checked and automatically retried upon.