Catch error and log warning when not able to update mtimes
When we launch an instance, nova updates the mtime for the _base image to let the image cache manager know the image is actively used. This can lead to unexpected I/O errors when launching a large amount of instances at once coming from the same _base image. This commit puts the execute call in a try, except block to catch possible errors. It also logs a warning with the path and error message. With this, at least once the update will succeed. Closes-Bug: #1621818 Co-Authored-By: Matt Riedemann <mriedem@us.ibm.com> Change-Id: I2fd1700aa4563a906eb574cbbe16caa63abae0d6 Signed-off-by: Joris S'heeren <joris.sheeren@combellgroup.com>
This commit is contained in:
parent
76923cffbb
commit
53da313a86
|
@ -724,3 +724,10 @@ disk size: 4.4M
|
||||||
{'properties': {'architecture': "X86_64"}})
|
{'properties': {'architecture': "X86_64"}})
|
||||||
image_arch = libvirt_utils.get_arch(image_meta)
|
image_arch = libvirt_utils.get_arch(image_meta)
|
||||||
self.assertEqual(arch.X86_64, image_arch)
|
self.assertEqual(arch.X86_64, image_arch)
|
||||||
|
|
||||||
|
def test_update_mtime_error(self):
|
||||||
|
with mock.patch.object(libvirt_utils, 'execute',
|
||||||
|
side_effect=processutils.ProcessExecutionError):
|
||||||
|
with mock.patch.object(libvirt_utils.LOG, 'warning') as mock_log:
|
||||||
|
libvirt_utils.update_mtime(mock.sentinel.path)
|
||||||
|
self.assertTrue(mock_log.called)
|
||||||
|
|
|
@ -31,6 +31,7 @@ from nova.compute import vm_mode
|
||||||
import nova.conf
|
import nova.conf
|
||||||
from nova.i18n import _
|
from nova.i18n import _
|
||||||
from nova.i18n import _LI
|
from nova.i18n import _LI
|
||||||
|
from nova.i18n import _LW
|
||||||
from nova import utils
|
from nova import utils
|
||||||
from nova.virt import images
|
from nova.virt import images
|
||||||
from nova.virt.libvirt import config as vconfig
|
from nova.virt.libvirt import config as vconfig
|
||||||
|
@ -245,7 +246,16 @@ def update_mtime(path):
|
||||||
|
|
||||||
:param path: File bump the mtime on
|
:param path: File bump the mtime on
|
||||||
"""
|
"""
|
||||||
|
try:
|
||||||
execute('touch', '-c', path, run_as_root=True)
|
execute('touch', '-c', path, run_as_root=True)
|
||||||
|
except processutils.ProcessExecutionError as exc:
|
||||||
|
# touch can intermittently fail when launching several instances with
|
||||||
|
# the same base image and using shared storage, so log the exception
|
||||||
|
# but don't fail. Ideally we'd know if we were on shared storage and
|
||||||
|
# would re-raise the error if we are not on shared storage.
|
||||||
|
LOG.warning(_LW("Failed to update mtime on path %(path)s. "
|
||||||
|
"Error: %(error)s"),
|
||||||
|
{'path': path, "error": exc})
|
||||||
|
|
||||||
|
|
||||||
def _id_map_to_config(id_map):
|
def _id_map_to_config(id_map):
|
||||||
|
|
Loading…
Reference in New Issue