Fix image cache periodic task concurrent access bug

This fix catch the exception and ignore a disk file absent
exception. It will defer the image cache clean to next round
image cache periodic task because it's not harmful.
Refer to detailed info, log analysis and root cause in defect.

Change-Id: I3a431bf389658f976edf5c847c04b7e0aac496fd
Closes-Bug: #1261442

nova/tests/virt/libvirt/test_imagecache.py

@@ -29,6 +29,7 @@ from nova import conductor
 from nova import db
 from nova.openstack.common import importutils
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import test
 from nova import utils
 from nova.virt.libvirt import imagecache
@@ -228,6 +229,24 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
         self.assertEqual(inuse_images, [found])
         self.assertEqual(len(image_cache_manager.unexplained_images), 0)
 
+    def test_list_backing_images_disk_notexist(self):
+        self.stubs.Set(os, 'listdir',
+                       lambda x: ['_base', 'banana-42-hamster'])
+        self.stubs.Set(os.path, 'exists',
+                       lambda x: x.find('banana-42-hamster') != -1)
+
+        def fake_get_disk(disk_path):
+            raise processutils.ProcessExecutionError()
+
+        self.stubs.Set(virtutils, 'get_disk_backing_file', fake_get_disk)
+
+        image_cache_manager = imagecache.ImageCacheManager()
+        image_cache_manager.unexplained_images = []
+        image_cache_manager.instance_names = self.stock_instance_names
+
+        self.assertRaises(processutils.ProcessExecutionError,
+                          image_cache_manager._list_backing_images)
+
     def test_find_base_file_nothing(self):
         self.stubs.Set(os.path, 'exists', lambda x: False)
 

nova/virt/libvirt/imagecache.py

@@ -34,6 +34,7 @@ from nova.openstack.common import fileutils
 from nova.openstack.common.gettextutils import _
 from nova.openstack.common import jsonutils
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import utils
 from nova.virt import imagecache
 from nova.virt.libvirt import utils as virtutils
@@ -297,7 +298,17 @@ class ImageCacheManager(imagecache.ImageCacheManager):
                 disk_path = os.path.join(CONF.instances_path, ent, 'disk')
                 if os.path.exists(disk_path):
                     LOG.debug(_('%s has a disk file'), ent)
-                    backing_file = virtutils.get_disk_backing_file(disk_path)
+                    try:
+                        backing_file = virtutils.get_disk_backing_file(
+                            disk_path)
+                    except processutils.ProcessExecutionError:
+                        # (for bug 1261442)
+                        if not os.path.exists(disk_path):
+                            LOG.debug(_('Failed to get disk backing file: %s'),
+                                      disk_path)
+                            continue
+                        else:
+                            raise
                     LOG.debug(_('Instance %(instance)s is backed by '
                                 '%(backing)s'),
                               {'instance': ent,