xenapi: cached images should be cleaned up by time

For xenapi driver, there needs to be some way to delete cached
images based on when they were created. add an optional arg to
control delete operation.

Change-Id: I24fc45e989aa951aab55a261fce77f7e3667d988
Closes-bug: 1481689

nova/tests/unit/virt/xenapi/test_vm_utils.py

@@ -46,6 +46,7 @@ from nova.virt import hardware
 from nova.virt.xenapi import driver as xenapi_conn
 from nova.virt.xenapi import fake
 from nova.virt.xenapi import vm_utils
+import time
 
 CONF = nova.conf.CONF
 XENSM_TYPE = 'xensm'
@@ -628,13 +629,119 @@ class CreateCachedImageTestCase(VMUtilsTestBase):
                        mock_safe_find_sr):
         self.session.call_xenapi.side_effect = ['ext', {}, 'cache_vdi_ref',
                                                 None, None, None, None, None,
-                                                None, 'vdi_uuid']
+                                                None, None, 'vdi_uuid']
         self.assertEqual((True, {'root': {'uuid': 'vdi_uuid', 'file': None}}),
                          vm_utils._create_cached_image('context', self.session,
                                     'instance', 'name', 'uuid',
                                     vm_utils.ImageType.DISK_VHD))
 
 
+class DestroyCachedImageTestCase(VMUtilsTestBase):
+    def setUp(self):
+        super(DestroyCachedImageTestCase, self).setUp()
+        self.session = stubs.get_fake_session()
+
+    @mock.patch.object(vm_utils, '_find_cached_images')
+    @mock.patch.object(vm_utils, 'destroy_vdi')
+    @mock.patch.object(vm_utils, '_walk_vdi_chain')
+    @mock.patch.object(time, 'time')
+    def test_destroy_cached_image_out_of_keep_days(self,
+                                                   mock_time,
+                                                   mock_walk_vdi_chain,
+                                                   mock_destroy_vdi,
+                                                   mock_find_cached_images):
+        fake_cached_time = '0'
+        mock_find_cached_images.return_value = {'fake_image_id': {
+            'vdi_ref': 'fake_vdi_ref', 'cached_time': fake_cached_time}}
+        self.session.call_xenapi.return_value = 'fake_uuid'
+        mock_walk_vdi_chain.return_value = ('just_one',)
+
+        mock_time.return_value = 2 * 3600 * 24
+
+        fake_keep_days = 1
+        expected_return = set()
+        expected_return.add('fake_uuid')
+
+        uuid_return = vm_utils.destroy_cached_images(self.session,
+            'fake_sr_ref', False, False, fake_keep_days)
+        mock_find_cached_images.assert_called_once()
+        mock_walk_vdi_chain.assert_called_once()
+        mock_time.assert_called()
+        mock_destroy_vdi.assert_called_once()
+        self.assertEqual(expected_return, uuid_return)
+
+    @mock.patch.object(vm_utils, '_find_cached_images')
+    @mock.patch.object(vm_utils, 'destroy_vdi')
+    @mock.patch.object(vm_utils, '_walk_vdi_chain')
+    @mock.patch.object(time, 'time')
+    def test_destroy_cached_image(self, mock_time, mock_walk_vdi_chain,
+                                  mock_destroy_vdi, mock_find_cached_images):
+        fake_cached_time = '0'
+        mock_find_cached_images.return_value = {'fake_image_id': {
+            'vdi_ref': 'fake_vdi_ref', 'cached_time': fake_cached_time}}
+        self.session.call_xenapi.return_value = 'fake_uuid'
+        mock_walk_vdi_chain.return_value = ('just_one',)
+
+        mock_time.return_value = 2 * 3600 * 24
+
+        fake_keep_days = 1
+        expected_return = set()
+        expected_return.add('fake_uuid')
+
+        uuid_return = vm_utils.destroy_cached_images(self.session,
+            'fake_sr_ref', False, False, fake_keep_days)
+        mock_find_cached_images.assert_called_once()
+        mock_walk_vdi_chain.assert_called_once()
+        mock_destroy_vdi.assert_called_once()
+        self.assertEqual(expected_return, uuid_return)
+
+    @mock.patch.object(vm_utils, '_find_cached_images')
+    @mock.patch.object(vm_utils, 'destroy_vdi')
+    @mock.patch.object(vm_utils, '_walk_vdi_chain')
+    @mock.patch.object(time, 'time')
+    def test_destroy_cached_image_cached_time_not_exceed(
+        self, mock_time, mock_walk_vdi_chain,
+        mock_destroy_vdi, mock_find_cached_images):
+        fake_cached_time = '0'
+        mock_find_cached_images.return_value = {'fake_image_id': {
+            'vdi_ref': 'fake_vdi_ref', 'cached_time': fake_cached_time}}
+        self.session.call_xenapi.return_value = 'fake_uuid'
+        mock_walk_vdi_chain.return_value = ('just_one',)
+
+        mock_time.return_value = 1 * 3600 * 24
+
+        fake_keep_days = 2
+        expected_return = set()
+
+        uuid_return = vm_utils.destroy_cached_images(self.session,
+            'fake_sr_ref', False, False, fake_keep_days)
+        mock_find_cached_images.assert_called_once()
+        mock_walk_vdi_chain.assert_called_once()
+        mock_destroy_vdi.assert_not_called()
+        self.assertEqual(expected_return, uuid_return)
+
+    @mock.patch.object(vm_utils, '_find_cached_images')
+    @mock.patch.object(vm_utils, 'destroy_vdi')
+    @mock.patch.object(vm_utils, '_walk_vdi_chain')
+    @mock.patch.object(time, 'time')
+    def test_destroy_cached_image_no_cached_time(
+        self, mock_time, mock_walk_vdi_chain,
+        mock_destroy_vdi, mock_find_cached_images):
+        mock_find_cached_images.return_value = {'fake_image_id': {
+            'vdi_ref': 'fake_vdi_ref', 'cached_time': None}}
+        self.session.call_xenapi.return_value = 'fake_uuid'
+        mock_walk_vdi_chain.return_value = ('just_one',)
+        fake_keep_days = 2
+        expected_return = set()
+
+        uuid_return = vm_utils.destroy_cached_images(self.session,
+            'fake_sr_ref', False, False, fake_keep_days)
+        mock_find_cached_images.assert_called_once()
+        mock_walk_vdi_chain.assert_called_once()
+        mock_destroy_vdi.assert_not_called()
+        self.assertEqual(expected_return, uuid_return)
+
+
 class ShutdownTestCase(VMUtilsTestBase):
 
     def test_hardshutdown_should_return_true_when_vm_is_shutdown(self):

nova/virt/xenapi/vm_utils.py

@@ -716,7 +716,8 @@ def get_sr_path(session, sr_ref=None):
     return os.path.join(CONF.xenserver.sr_base_path, sr_uuid)
 
 
-def destroy_cached_images(session, sr_ref, all_cached=False, dry_run=False):
+def destroy_cached_images(session, sr_ref, all_cached=False, dry_run=False,
+                          keep_days=0):
     """Destroy used or unused cached images.
 
     A cached image that is being used by at least one VM is said to be 'used'.
@@ -728,6 +729,10 @@ def destroy_cached_images(session, sr_ref, all_cached=False, dry_run=False):
 
     The default behavior of this function is to destroy only 'unused' cached
     images. To destroy all cached images, use the `all_cached=True` kwarg.
+
+    `keep_days` is used to destroy images based on when they were created.
+    Only the images which were created `keep_days` ago will be deleted if the
+    argument has been set.
     """
     cached_images = _find_cached_images(session, sr_ref)
     destroyed = set()
@@ -738,7 +743,8 @@ def destroy_cached_images(session, sr_ref, all_cached=False, dry_run=False):
             destroy_vdi(session, vdi_ref)
         destroyed.add(vdi_uuid)
 
-    for vdi_ref in cached_images.values():
+    for vdi_dict in cached_images.values():
+        vdi_ref = vdi_dict['vdi_ref']
         vdi_uuid = session.call_xenapi('VDI.get_uuid', vdi_ref)
 
         if all_cached:
@@ -760,13 +766,22 @@ def destroy_cached_images(session, sr_ref, all_cached=False, dry_run=False):
             if len(children) > 1:
                 continue
 
-        destroy_cached_vdi(vdi_uuid, vdi_ref)
+        cached_time = vdi_dict.get('cached_time')
+        if cached_time is not None:
+            if (int(time.time()) - int(cached_time)) / (3600 * 24) \
+               >= keep_days:
+                destroy_cached_vdi(vdi_uuid, vdi_ref)
+        else:
+            LOG.debug("vdi %s can't be destroyed because the cached time is"
+                      " not specified", vdi_uuid)
 
     return destroyed
 
 
 def _find_cached_images(session, sr_ref):
-    """Return a dict(uuid=vdi_ref) representing all cached images."""
+    """Return a dict {image_id: {'vdi_ref': vdi_ref, 'cached_time':
+    cached_time}} representing all cached images.
+    """
     cached_images = {}
     for vdi_ref, vdi_rec in _get_all_vdis_in_sr(session, sr_ref):
         try:
@@ -774,7 +789,9 @@ def _find_cached_images(session, sr_ref):
         except KeyError:
             continue
 
-        cached_images[image_id] = vdi_ref
+        cached_time = vdi_rec['other_config'].get('cached-time')
+        cached_images[image_id] = {'vdi_ref': vdi_ref,
+                                   'cached_time': cached_time}
 
     return cached_images
 
@@ -1226,6 +1243,10 @@ def _create_cached_image(context, session, instance, name_label,
                                 'root')
             session.call_xenapi('VDI.add_to_other_config',
                                 cache_vdi_ref, 'image-id', str(image_id))
+            session.call_xenapi('VDI.add_to_other_config',
+                                cache_vdi_ref,
+                                'cached-time',
+                                str(int(time.time())))
 
         if CONF.use_cow_images:
             new_vdi_ref = _clone_vdi(session, cache_vdi_ref)

releasenotes/notes/xentool-destory-cached-image-c9d39a733002ca7d.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    For the XenAPI driver, in order to delete cached images based on when they
+    were created, a new ``--keep-days DAYS`` option is added to the
+    ``destroy_cached_images`` script to delete cached images which were created
+    at least ``DAYS`` days ago. By default, all unused cached images will be
+    deleted when the script is run if they have ``cached_time``.

tools/xenserver/destroy_cached_images.py

@@ -21,6 +21,8 @@ Options:
     --dry_run - Don't actually destroy the VDIs
     --all_cached - Destroy all cached images instead of just unused cached
                    images.
+    --keep_days - N - Only remove those cached images which were created
+                      more than N days ago.
 """
 import eventlet
 eventlet.monkey_patch()
@@ -52,7 +54,11 @@ destroy_opts = [
                      ' images.'),
     cfg.BoolOpt('dry_run',
                 default=False,
-                help='Don\'t actually delete the VDIs.')
+                help='Don\'t actually delete the VDIs.'),
+    cfg.IntOpt('keep_days',
+               default=0,
+               help='Destroy cached images which were'
+                    ' created over keep_days.')
 ]
 
 CONF = nova.conf.CONF
@@ -69,7 +75,7 @@ def main():
     sr_ref = vm_utils.safe_find_sr(_session)
     destroyed = vm_utils.destroy_cached_images(
         _session, sr_ref, all_cached=CONF.all_cached,
-        dry_run=CONF.dry_run)
+        dry_run=CONF.dry_run, keep_days=CONF.keep_days)
 
     if '--verbose' in sys.argv:
         print('\n'.join(destroyed))