openstack
/
nova
Code Issues Proposed changes

Catch error and log warning when not able to update mtimes 

When we launch an instance, nova updates the mtime for the _base image
to let the image cache manager know the image is actively used.  This
can lead to unexpected I/O errors when launching a large amount of
instances at once coming from the same _base image.

This commit puts the execute call in a try, except block to catch
possible errors. It also logs a warning with the path and error message.
With this, at least once the update will succeed.

Closes-Bug: #1621818

Co-Authored-By: Matt Riedemann <mriedem@us.ibm.com>

Change-Id: I2fd1700aa4563a906eb574cbbe16caa63abae0d6
Signed-off-by: Joris S'heeren <joris.sheeren@combellgroup.com>
This commit is contained in:
Joris S'heeren 2016-09-09 15:40:58 +02:00 committed by Matt Riedemann
parent 76923cffbb
commit 53da313a86
2 changed files with 18 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
nova/tests/unit/virt/libvirt/test_utils.py
View File

@ -724,3 +724,10 @@ disk size: 4.4M
{'properties': {'architecture': "X86_64"}})
image_arch = libvirt_utils.get_arch(image_meta)
self.assertEqual(arch.X86_64, image_arch)
def test_update_mtime_error(self):
with mock.patch.object(libvirt_utils, 'execute',
side_effect=processutils.ProcessExecutionError):
with mock.patch.object(libvirt_utils.LOG, 'warning') as mock_log:
libvirt_utils.update_mtime(mock.sentinel.path)
self.assertTrue(mock_log.called)

12
nova/virt/libvirt/utils.py
View File

@ -31,6 +31,7 @@ from nova.compute import vm_mode
import nova.conf
from nova.i18n import _
from nova.i18n import _LI
from nova.i18n import _LW
from nova import utils
from nova.virt import images
from nova.virt.libvirt import config as vconfig
@ -245,7 +246,16 @@ def update_mtime(path):
:param path: File bump the mtime on
"""
execute('touch', '-c', path, run_as_root=True)
try:
execute('touch', '-c', path, run_as_root=True)
except processutils.ProcessExecutionError as exc:
# touch can intermittently fail when launching several instances with
# the same base image and using shared storage, so log the exception
# but don't fail. Ideally we'd know if we were on shared storage and
# would re-raise the error if we are not on shared storage.
LOG.warning(_LW("Failed to update mtime on path %(path)s. "
"Error: %(error)s"),
{'path': path, "error": exc})
def _id_map_to_config(id_map):