From 693ace79fbf856967684f11dfd7663d465dbe19a Mon Sep 17 00:00:00 2001 From: naichuans Date: Thu, 18 May 2017 13:01:06 +0000 Subject: [PATCH] xenapi: cached images should be cleaned up by time For xenapi driver, there needs to be some way to delete cached images based on when they were created. add an optional arg to control delete operation. Change-Id: I24fc45e989aa951aab55a261fce77f7e3667d988 Closes-bug: 1481689 --- nova/tests/unit/virt/xenapi/test_vm_utils.py | 109 +++++++++++++++++- nova/virt/xenapi/vm_utils.py | 31 ++++- ...destory-cached-image-c9d39a733002ca7d.yaml | 8 ++ tools/xenserver/destroy_cached_images.py | 10 +- 4 files changed, 150 insertions(+), 8 deletions(-) create mode 100644 releasenotes/notes/xentool-destory-cached-image-c9d39a733002ca7d.yaml diff --git a/nova/tests/unit/virt/xenapi/test_vm_utils.py b/nova/tests/unit/virt/xenapi/test_vm_utils.py index ea66f45b2e6f..3b5c97404633 100644 --- a/nova/tests/unit/virt/xenapi/test_vm_utils.py +++ b/nova/tests/unit/virt/xenapi/test_vm_utils.py @@ -46,6 +46,7 @@ from nova.virt import hardware from nova.virt.xenapi import driver as xenapi_conn from nova.virt.xenapi import fake from nova.virt.xenapi import vm_utils +import time CONF = nova.conf.CONF XENSM_TYPE = 'xensm' @@ -628,13 +629,119 @@ class CreateCachedImageTestCase(VMUtilsTestBase): mock_safe_find_sr): self.session.call_xenapi.side_effect = ['ext', {}, 'cache_vdi_ref', None, None, None, None, None, - None, 'vdi_uuid'] + None, None, 'vdi_uuid'] self.assertEqual((True, {'root': {'uuid': 'vdi_uuid', 'file': None}}), vm_utils._create_cached_image('context', self.session, 'instance', 'name', 'uuid', vm_utils.ImageType.DISK_VHD)) +class DestroyCachedImageTestCase(VMUtilsTestBase): + def setUp(self): + super(DestroyCachedImageTestCase, self).setUp() + self.session = stubs.get_fake_session() + + @mock.patch.object(vm_utils, '_find_cached_images') + @mock.patch.object(vm_utils, 'destroy_vdi') + @mock.patch.object(vm_utils, '_walk_vdi_chain') + @mock.patch.object(time, 'time') + def test_destroy_cached_image_out_of_keep_days(self, + mock_time, + mock_walk_vdi_chain, + mock_destroy_vdi, + mock_find_cached_images): + fake_cached_time = '0' + mock_find_cached_images.return_value = {'fake_image_id': { + 'vdi_ref': 'fake_vdi_ref', 'cached_time': fake_cached_time}} + self.session.call_xenapi.return_value = 'fake_uuid' + mock_walk_vdi_chain.return_value = ('just_one',) + + mock_time.return_value = 2 * 3600 * 24 + + fake_keep_days = 1 + expected_return = set() + expected_return.add('fake_uuid') + + uuid_return = vm_utils.destroy_cached_images(self.session, + 'fake_sr_ref', False, False, fake_keep_days) + mock_find_cached_images.assert_called_once() + mock_walk_vdi_chain.assert_called_once() + mock_time.assert_called() + mock_destroy_vdi.assert_called_once() + self.assertEqual(expected_return, uuid_return) + + @mock.patch.object(vm_utils, '_find_cached_images') + @mock.patch.object(vm_utils, 'destroy_vdi') + @mock.patch.object(vm_utils, '_walk_vdi_chain') + @mock.patch.object(time, 'time') + def test_destroy_cached_image(self, mock_time, mock_walk_vdi_chain, + mock_destroy_vdi, mock_find_cached_images): + fake_cached_time = '0' + mock_find_cached_images.return_value = {'fake_image_id': { + 'vdi_ref': 'fake_vdi_ref', 'cached_time': fake_cached_time}} + self.session.call_xenapi.return_value = 'fake_uuid' + mock_walk_vdi_chain.return_value = ('just_one',) + + mock_time.return_value = 2 * 3600 * 24 + + fake_keep_days = 1 + expected_return = set() + expected_return.add('fake_uuid') + + uuid_return = vm_utils.destroy_cached_images(self.session, + 'fake_sr_ref', False, False, fake_keep_days) + mock_find_cached_images.assert_called_once() + mock_walk_vdi_chain.assert_called_once() + mock_destroy_vdi.assert_called_once() + self.assertEqual(expected_return, uuid_return) + + @mock.patch.object(vm_utils, '_find_cached_images') + @mock.patch.object(vm_utils, 'destroy_vdi') + @mock.patch.object(vm_utils, '_walk_vdi_chain') + @mock.patch.object(time, 'time') + def test_destroy_cached_image_cached_time_not_exceed( + self, mock_time, mock_walk_vdi_chain, + mock_destroy_vdi, mock_find_cached_images): + fake_cached_time = '0' + mock_find_cached_images.return_value = {'fake_image_id': { + 'vdi_ref': 'fake_vdi_ref', 'cached_time': fake_cached_time}} + self.session.call_xenapi.return_value = 'fake_uuid' + mock_walk_vdi_chain.return_value = ('just_one',) + + mock_time.return_value = 1 * 3600 * 24 + + fake_keep_days = 2 + expected_return = set() + + uuid_return = vm_utils.destroy_cached_images(self.session, + 'fake_sr_ref', False, False, fake_keep_days) + mock_find_cached_images.assert_called_once() + mock_walk_vdi_chain.assert_called_once() + mock_destroy_vdi.assert_not_called() + self.assertEqual(expected_return, uuid_return) + + @mock.patch.object(vm_utils, '_find_cached_images') + @mock.patch.object(vm_utils, 'destroy_vdi') + @mock.patch.object(vm_utils, '_walk_vdi_chain') + @mock.patch.object(time, 'time') + def test_destroy_cached_image_no_cached_time( + self, mock_time, mock_walk_vdi_chain, + mock_destroy_vdi, mock_find_cached_images): + mock_find_cached_images.return_value = {'fake_image_id': { + 'vdi_ref': 'fake_vdi_ref', 'cached_time': None}} + self.session.call_xenapi.return_value = 'fake_uuid' + mock_walk_vdi_chain.return_value = ('just_one',) + fake_keep_days = 2 + expected_return = set() + + uuid_return = vm_utils.destroy_cached_images(self.session, + 'fake_sr_ref', False, False, fake_keep_days) + mock_find_cached_images.assert_called_once() + mock_walk_vdi_chain.assert_called_once() + mock_destroy_vdi.assert_not_called() + self.assertEqual(expected_return, uuid_return) + + class ShutdownTestCase(VMUtilsTestBase): def test_hardshutdown_should_return_true_when_vm_is_shutdown(self): diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 7e1bcf8eaae3..43fcee8fc5b1 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -716,7 +716,8 @@ def get_sr_path(session, sr_ref=None): return os.path.join(CONF.xenserver.sr_base_path, sr_uuid) -def destroy_cached_images(session, sr_ref, all_cached=False, dry_run=False): +def destroy_cached_images(session, sr_ref, all_cached=False, dry_run=False, + keep_days=0): """Destroy used or unused cached images. A cached image that is being used by at least one VM is said to be 'used'. @@ -728,6 +729,10 @@ def destroy_cached_images(session, sr_ref, all_cached=False, dry_run=False): The default behavior of this function is to destroy only 'unused' cached images. To destroy all cached images, use the `all_cached=True` kwarg. + + `keep_days` is used to destroy images based on when they were created. + Only the images which were created `keep_days` ago will be deleted if the + argument has been set. """ cached_images = _find_cached_images(session, sr_ref) destroyed = set() @@ -738,7 +743,8 @@ def destroy_cached_images(session, sr_ref, all_cached=False, dry_run=False): destroy_vdi(session, vdi_ref) destroyed.add(vdi_uuid) - for vdi_ref in cached_images.values(): + for vdi_dict in cached_images.values(): + vdi_ref = vdi_dict['vdi_ref'] vdi_uuid = session.call_xenapi('VDI.get_uuid', vdi_ref) if all_cached: @@ -760,13 +766,22 @@ def destroy_cached_images(session, sr_ref, all_cached=False, dry_run=False): if len(children) > 1: continue - destroy_cached_vdi(vdi_uuid, vdi_ref) + cached_time = vdi_dict.get('cached_time') + if cached_time is not None: + if (int(time.time()) - int(cached_time)) / (3600 * 24) \ + >= keep_days: + destroy_cached_vdi(vdi_uuid, vdi_ref) + else: + LOG.debug("vdi %s can't be destroyed because the cached time is" + " not specified", vdi_uuid) return destroyed def _find_cached_images(session, sr_ref): - """Return a dict(uuid=vdi_ref) representing all cached images.""" + """Return a dict {image_id: {'vdi_ref': vdi_ref, 'cached_time': + cached_time}} representing all cached images. + """ cached_images = {} for vdi_ref, vdi_rec in _get_all_vdis_in_sr(session, sr_ref): try: @@ -774,7 +789,9 @@ def _find_cached_images(session, sr_ref): except KeyError: continue - cached_images[image_id] = vdi_ref + cached_time = vdi_rec['other_config'].get('cached-time') + cached_images[image_id] = {'vdi_ref': vdi_ref, + 'cached_time': cached_time} return cached_images @@ -1226,6 +1243,10 @@ def _create_cached_image(context, session, instance, name_label, 'root') session.call_xenapi('VDI.add_to_other_config', cache_vdi_ref, 'image-id', str(image_id)) + session.call_xenapi('VDI.add_to_other_config', + cache_vdi_ref, + 'cached-time', + str(int(time.time()))) if CONF.use_cow_images: new_vdi_ref = _clone_vdi(session, cache_vdi_ref) diff --git a/releasenotes/notes/xentool-destory-cached-image-c9d39a733002ca7d.yaml b/releasenotes/notes/xentool-destory-cached-image-c9d39a733002ca7d.yaml new file mode 100644 index 000000000000..b19141f6f446 --- /dev/null +++ b/releasenotes/notes/xentool-destory-cached-image-c9d39a733002ca7d.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + For the XenAPI driver, in order to delete cached images based on when they + were created, a new ``--keep-days DAYS`` option is added to the + ``destroy_cached_images`` script to delete cached images which were created + at least ``DAYS`` days ago. By default, all unused cached images will be + deleted when the script is run if they have ``cached_time``. diff --git a/tools/xenserver/destroy_cached_images.py b/tools/xenserver/destroy_cached_images.py index 1cf254788d45..c9025e9423ae 100644 --- a/tools/xenserver/destroy_cached_images.py +++ b/tools/xenserver/destroy_cached_images.py @@ -21,6 +21,8 @@ Options: --dry_run - Don't actually destroy the VDIs --all_cached - Destroy all cached images instead of just unused cached images. + --keep_days - N - Only remove those cached images which were created + more than N days ago. """ import eventlet eventlet.monkey_patch() @@ -52,7 +54,11 @@ destroy_opts = [ ' images.'), cfg.BoolOpt('dry_run', default=False, - help='Don\'t actually delete the VDIs.') + help='Don\'t actually delete the VDIs.'), + cfg.IntOpt('keep_days', + default=0, + help='Destroy cached images which were' + ' created over keep_days.') ] CONF = nova.conf.CONF @@ -69,7 +75,7 @@ def main(): sr_ref = vm_utils.safe_find_sr(_session) destroyed = vm_utils.destroy_cached_images( _session, sr_ref, all_cached=CONF.all_cached, - dry_run=CONF.dry_run) + dry_run=CONF.dry_run, keep_days=CONF.keep_days) if '--verbose' in sys.argv: print('\n'.join(destroyed))