openstack/nova
Code Issues Proposed changes

Consolidate [image_cache] conf options

Browse Source 
Blueprint image-precache-support added a conf section called
[image_cache], so it makes sense to move all the existing image
cache-related conf options into it.

Old:
[DEFAULT]image_cache_manager_interval
[DEFAULT]image_cache_subdirectory_name
[DEFAULT]remove_unused_base_images
[DEFAULT]remove_unused_original_minimum_age_seconds
[libvirt]remove_unused_resized_minimum_age_seconds

New:
[image_cache]manager_interval
[image_cache]subdirectory_name
[image_cache]remove_unused_base_images
[image_cache]remove_unused_original_minimum_age_seconds
[image_cache]remove_unused_resized_minimum_age_seconds

Change-Id: I3c49825ac0d70152b6c8ee4c8ca01546265f4b80
Partial-Bug: #1847302
This commit is contained in:
Eric Fried 2019-10-23 14:41:36 -05:00
parent dcfd74fb37
commit 828e8047e5
21 changed files with 132 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
doc/source/admin/configuration/hypervisor-vmware.rst
View File

@ -926,18 +926,11 @@ cached images are stored.
have a shared file system.
You can automatically purge unused images after a specified period of time. To
configure this action, set these options in the ``DEFAULT`` section in the
``nova.conf`` file:
configure this action, set these options in the :oslo.config:group`image_cache`
section in the ``nova.conf`` file:
``remove_unused_base_images``
Set this option to ``True`` to specify that unused images should be removed
after the duration specified in the
``remove_unused_original_minimum_age_seconds`` option. The default is
``True``.
``remove_unused_original_minimum_age_seconds``
Specifies the duration in seconds after which an unused image is purged from
the cache. The default is ``86400`` (24 hours).
* :oslo.config:option:`image_cache.remove_unused_base_images`
* :oslo.config:option:`image_cache.remove_unused_original_minimum_age_seconds`
.. _vmware-networking:

9
doc/source/admin/image-caching.rst
View File

@ -38,13 +38,8 @@ default). Those unused images are deleted from the cache directory
until they are needed again.
For more information about configuring image cache behavior, see the
documentation for the following configuration options:
- :oslo.config:option:`image_cache_subdirectory_name`
- :oslo.config:option:`image_cache_manager_interval`
- :oslo.config:option:`remove_unused_base_images`
- :oslo.config:option:`remove_unused_original_minimum_age_seconds`
- :oslo.config:option:`image_cache.precache_concurrency`
documentation for the configuration options in the
:oslo.config:group:`image_cache` group.
.. note::

2
nova/compute/manager.py
View File

@ -9698,7 +9698,7 @@ class ComputeManager(manager.Manager):
else:
self._process_instance_event(instance, event)
@periodic_task.periodic_task(spacing=CONF.image_cache_manager_interval,
@periodic_task.periodic_task(spacing=CONF.image_cache.manager_interval,
external_process_ok=True)
def _run_image_cache_manager_pass(self, context):
"""Run a single pass of the image cache manager."""

41
nova/conf/compute.py
View File

@ -264,22 +264,6 @@ driver (for qcow and raw), or loop (for raw).
default=10,
min=0,
help='Amount of time, in seconds, to wait for NBD device start up.'),
cfg.StrOpt('image_cache_subdirectory_name',
default='_base',
help="""
Location of cached images.
This is NOT the full path - just a folder name relative to '$instances_path'.
For per-compute-host cached images, set to '_base_$my_ip'
"""),
cfg.BoolOpt('remove_unused_base_images',
default=True,
help='Should unused base images be removed?'),
cfg.IntOpt('remove_unused_original_minimum_age_seconds',
default=(24 * 3600),
help="""
Unused unresized base images younger than this will not be removed.
"""),
cfg.StrOpt('pointer_model',
default='usbtablet',
choices=[
@ -954,31 +938,6 @@ Possible values:
]
interval_opts = [
cfg.IntOpt('image_cache_manager_interval',
default=2400,
min=-1,
help="""
Number of seconds to wait between runs of the image cache manager.
Note that when using shared storage for the ``[DEFAULT]/instances_path``
configuration option across multiple nova-compute services, this periodic
could process a large number of instances. Similarly, using a compute driver
that manages a cluster (like vmwareapi.VMwareVCDriver) could result in
processing a large number of instances. Therefore you may need to adjust the
time interval for the anticipated load, or only run on one nova-compute
service within a shared storage aggregate.
Possible values:
* 0: run at the default interval of 60 seconds (not recommended)
* -1: disable
* Any other value
Related options:
* ``[DEFAULT]/compute_driver``
* ``[DEFAULT]/instances_path``
"""),
cfg.IntOpt('bandwidth_poll_interval',
default=600,
help="""

53
nova/conf/imagecache.py
View File

@ -19,6 +19,59 @@ imagecache_group = cfg.OptGroup(
A collection of options specific to image caching.
""")
imagecache_opts = [
cfg.IntOpt('manager_interval',
default=2400,
min=-1,
deprecated_name='image_cache_manager_interval',
deprecated_group='DEFAULT',
help="""
Number of seconds to wait between runs of the image cache manager.
Note that when using shared storage for the ``[DEFAULT]/instances_path``
configuration option across multiple nova-compute services, this periodic
could process a large number of instances. Similarly, using a compute driver
that manages a cluster (like vmwareapi.VMwareVCDriver) could result in
processing a large number of instances. Therefore you may need to adjust the
time interval for the anticipated load, or only run on one nova-compute
service within a shared storage aggregate.
Possible values:
* 0: run at the default interval of 60 seconds (not recommended)
* -1: disable
* Any other value
Related options:
* ``[DEFAULT]/compute_driver``
* ``[DEFAULT]/instances_path``
"""),
cfg.StrOpt('subdirectory_name',
default='_base',
deprecated_name='image_cache_subdirectory_name',
deprecated_group='DEFAULT',
help="""
Location of cached images.
This is NOT the full path - just a folder name relative to '$instances_path'.
For per-compute-host cached images, set to '_base_$my_ip'
"""),
cfg.BoolOpt('remove_unused_base_images',
default=True,
deprecated_group='DEFAULT',
help='Should unused base images be removed?'),
cfg.IntOpt('remove_unused_original_minimum_age_seconds',
default=(24 * 3600),
deprecated_group='DEFAULT',
help="""
Unused unresized base images younger than this will not be removed.
"""),
cfg.IntOpt('remove_unused_resized_minimum_age_seconds',
default=3600,
deprecated_group='libvirt',
help="""
Unused resized base images younger than this will not be removed.
"""),
cfg.IntOpt('precache_concurrency',
default=1,
min=1,

8
nova/conf/libvirt.py
View File

@ -953,13 +953,6 @@ Requires:
"""),
]
libvirt_imagecache_opts = [
cfg.IntOpt('remove_unused_resized_minimum_age_seconds',
default=3600,
help='Unused resized base images younger than this will not be '
'removed'),
]
libvirt_lvm_opts = [
cfg.StrOpt('volume_clear',
default='zero',
@ -1364,7 +1357,6 @@ For example::
ALL_OPTS = list(itertools.chain(
libvirt_general_opts,
libvirt_imagebackend_opts,
libvirt_imagecache_opts,
libvirt_lvm_opts,
libvirt_utils_opts,
libvirt_vif_opts,

3
nova/tests/unit/virt/hyperv/test_imagecache.py
View File

@ -188,7 +188,8 @@ class ImageCacheTestCase(test_base.HyperVBaseTestCase):
@ddt.data(True, False)
def test_age_and_verify_cached_images(self, remove_unused_base_images):
self.flags(remove_unused_base_images=remove_unused_base_images)
self.flags(remove_unused_base_images=remove_unused_base_images,
group='image_cache')
fake_images = [mock.sentinel.FAKE_IMG1, mock.sentinel.FAKE_IMG2]
fake_used_images = [mock.sentinel.FAKE_IMG1]

10
nova/tests/unit/virt/libvirt/test_driver.py
View File

@ -739,7 +739,7 @@ class CacheConcurrencyTestCase(test.NoDBTestCase):
def fake_exists(fname):
basedir = os.path.join(CONF.instances_path,
CONF.image_cache_subdirectory_name)
CONF.image_cache.subdirectory_name)
if fname == basedir or fname == self.lock_path:
return True
return False
@ -12736,7 +12736,7 @@ class LibvirtConnTestCase(test.NoDBTestCase,
conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
base_dir = os.path.join(CONF.instances_path,
CONF.image_cache_subdirectory_name)
CONF.image_cache.subdirectory_name)
trusted_certs = objects.TrustedCerts(
ids=['0b5d2c72-12cc-4ba6-a8d7-3ff5cc1d8cb8',
'674736e3-f25c-405c-8362-bbf991e0ce0a'])
@ -12847,7 +12847,7 @@ class LibvirtConnTestCase(test.NoDBTestCase,
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
base_dir = os.path.join(CONF.instances_path,
CONF.image_cache_subdirectory_name)
CONF.image_cache.subdirectory_name)
instance = objects.Instance(**self.test_instance)
disk_info_byname = fake_disk_info_byname(instance)
@ -23578,7 +23578,7 @@ class LibvirtDriverTestCase(test.NoDBTestCase, TraitsComparisonMixin):
# are fully mocked out and are just testing string formatting in this
# test.
self.flags(instances_path='/nova/instances')
self.flags(image_cache_subdirectory_name='cache')
self.flags(subdirectory_name='cache', group='image_cache')
expected_fn = os.path.join('/nova/instances/cache',
imagecache.get_cache_fname('an-image'))
@ -23612,7 +23612,7 @@ class LibvirtDriverTestCase(test.NoDBTestCase, TraitsComparisonMixin):
# are fully mocked out and are just testing string formatting in this
# test.
self.flags(instances_path='/nova/instances')
self.flags(image_cache_subdirectory_name='cache')
self.flags(subdirectory_name='cache', group='image_cache')
expected_fn = os.path.join('/nova/instances/cache',
imagecache.get_cache_fname('an-image'))

2
nova/tests/unit/virt/libvirt/test_imagebackend.py
View File

@ -306,7 +306,7 @@ class FlatTestCase(_ImageTestCase, test.NoDBTestCase):
# a non-existent backend.
base_dir = os.path.join(CONF.instances_path,
CONF.image_cache_subdirectory_name)
CONF.image_cache.subdirectory_name)
# Lets assume the base image cache directory already exists
existing = set([base_dir])

10
nova/tests/unit/virt/libvirt/test_imagecache.py
View File

@ -160,7 +160,7 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
lambda x: 'e97222e91fc4241f49a7f520d1dcf446751129b3_sm')
found = os.path.join(CONF.instances_path,
CONF.image_cache_subdirectory_name,
CONF.image_cache.subdirectory_name,
'e97222e91fc4241f49a7f520d1dcf446751129b3_sm')
image_cache_manager = imagecache.ImageCacheManager()
@ -183,7 +183,7 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
'10737418240'))
found = os.path.join(CONF.instances_path,
CONF.image_cache_subdirectory_name,
CONF.image_cache.subdirectory_name,
'e97222e91fc4241f49a7f520d1dcf446751129b3_'
'10737418240')
@ -205,7 +205,7 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
lambda x: 'e97222e91fc4241f49a7f520d1dcf446751129b3_sm')
found = os.path.join(CONF.instances_path,
CONF.image_cache_subdirectory_name,
CONF.image_cache.subdirectory_name,
'e97222e91fc4241f49a7f520d1dcf446751129b3_sm')
image_cache_manager = imagecache.ImageCacheManager()
@ -415,8 +415,8 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
hashed_22 = '12c6fc06c99a462375eeb3f43dfd832b08ca9e17'
hashed_42 = '92cfceb39d57d914ed8b14d0e37643de0797ae56'
self.flags(instances_path='/instance_path',
image_cache_subdirectory_name='_base')
self.flags(instances_path='/instance_path')
self.flags(subdirectory_name='_base', group='image_cache')
base_file_list = ['00000001',
'ephemeral_0_20_None',

11
nova/tests/unit/virt/test_imagecache.py
View File

@ -53,11 +53,12 @@ swap_bdm_256 = [block_device.BlockDeviceDict(
class ImageCacheManagerTests(test.NoDBTestCase):
def test_configurationi_defaults(self):
self.assertEqual(2400, CONF.image_cache_manager_interval)
self.assertEqual('_base', CONF.image_cache_subdirectory_name)
self.assertTrue(CONF.remove_unused_base_images)
self.assertEqual(24 * 3600,
CONF.remove_unused_original_minimum_age_seconds)
self.assertEqual(2400, CONF.image_cache.manager_interval)
self.assertEqual('_base', CONF.image_cache.subdirectory_name)
self.assertTrue(CONF.image_cache.remove_unused_base_images)
self.assertEqual(
24 * 3600,
CONF.image_cache.remove_unused_original_minimum_age_seconds)
def test_cache_manager(self):
cache_manager = imagecache.ImageCacheManager()

9
nova/tests/unit/virt/vmwareapi/test_driver_api.py
View File

@ -213,8 +213,8 @@ class VMwareAPIVMTestCase(test.NoDBTestCase,
api_retry_count=1,
use_linked_clone=False, group='vmware')
self.flags(enabled=False, group='vnc')
self.flags(image_cache_subdirectory_name='vmware_base',
my_ip='')
self.flags(subdirectory_name='vmware_base', group='image_cache')
self.flags(my_ip='')
self.user_id = 'fake'
self.project_id = 'fake'
self.context = context.RequestContext(self.user_id, self.project_id)
@ -2021,7 +2021,7 @@ class VMwareAPIVMTestCase(test.NoDBTestCase,
def test_image_aging_disabled(self):
self._override_time()
self.flags(remove_unused_base_images=False)
self.flags(remove_unused_base_images=False, group='image_cache')
self._create_vm()
self._cached_files_exist()
all_instances = []
@ -2032,7 +2032,8 @@ class VMwareAPIVMTestCase(test.NoDBTestCase,
def _image_aging_aged(self, aging_time=100):
self._override_time()
cur_time = datetime.datetime(2012, 11, 22, 12, 00, 10)
self.flags(remove_unused_original_minimum_age_seconds=aging_time)
self.flags(remove_unused_original_minimum_age_seconds=aging_time,
group='image_cache')
self._image_aging_image_marked_for_deletion()
all_instances = []
self.useFixture(utils_fixture.TimeFixture(cur_time))

12
nova/tests/unit/virt/vmwareapi/test_vmops.py
View File

@ -64,8 +64,8 @@ class VMwareVMOpsTestCase(test.NoDBTestCase):
vmwareapi_fake.reset()
stubs.set_stubs(self)
self.flags(enabled=True, group='vnc')
self.flags(image_cache_subdirectory_name='vmware_base',
my_ip='',
self.flags(subdirectory_name='vmware_base', group='image_cache')
self.flags(my_ip='',
flat_injected=True)
self._context = context.RequestContext('fake_user', 'fake_project')
self._session = driver.VMwareAPISession()
@ -2723,19 +2723,19 @@ class VMwareVMOpsTestCase(test.NoDBTestCase):
self.assertEqual('flavor-policy', extra_specs.storage_policy)
def test_get_base_folder_not_set(self):
self.flags(image_cache_subdirectory_name='vmware_base')
self.flags(subdirectory_name='vmware_base', group='image_cache')
base_folder = self._vmops._get_base_folder()
self.assertEqual('vmware_base', base_folder)
def test_get_base_folder_host_ip(self):
self.flags(my_ip='7.7.7.7',
image_cache_subdirectory_name='_base')
self.flags(my_ip='7.7.7.7')
self.flags(subdirectory_name='_base', group='image_cache')
base_folder = self._vmops._get_base_folder()
self.assertEqual('7.7.7.7_base', base_folder)
def test_get_base_folder_cache_prefix(self):
self.flags(cache_prefix='my_prefix', group='vmware')
self.flags(image_cache_subdirectory_name='_base')
self.flags(subdirectory_name='_base', group='image_cache')
base_folder = self._vmops._get_base_folder()
self.assertEqual('my_prefix_base', base_folder)

5
nova/virt/hyperv/imagecache.py
View File

@ -171,7 +171,7 @@ class ImageCache(imagecache.ImageCacheManager):
# change the timestamp on the image so as to reflect the last
# time it was used
self._update_image_timestamp(img)
elif CONF.remove_unused_base_images:
elif CONF.image_cache.remove_unused_base_images:
self._remove_if_old_image(img)
def _update_image_timestamp(self, image):
@ -196,7 +196,8 @@ class ImageCache(imagecache.ImageCacheManager):
def _remove_if_old_image(self, image):
backing_files = self._get_image_backing_files(image)
max_age_seconds = CONF.remove_unused_original_minimum_age_seconds
max_age_seconds = (
CONF.image_cache.remove_unused_original_minimum_age_seconds)
for img in backing_files:
age_seconds = self._pathutils.get_age_of_file(img)

3
nova/virt/imagecache.py
View File

@ -29,7 +29,8 @@ class ImageCacheManager(object):
"""
def __init__(self):
self.remove_unused_base_images = CONF.remove_unused_base_images
self.remove_unused_base_images = (
CONF.image_cache.remove_unused_base_images)
self.resize_states = [task_states.RESIZE_PREP,
task_states.RESIZE_MIGRATING,
task_states.RESIZE_MIGRATED,

2
nova/virt/libvirt/driver.py
View File

@ -9658,7 +9658,7 @@ class LibvirtDriver(driver.ComputeDriver):
def cache_image(self, context, image_id):
cache_dir = os.path.join(CONF.instances_path,
CONF.image_cache_subdirectory_name)
CONF.image_cache.subdirectory_name)
path = os.path.join(cache_dir,
imagecache.get_cache_fname(image_id))
if os.path.exists(path):

2
nova/virt/libvirt/imagebackend.py
View File

@ -249,7 +249,7 @@ class Image(object):
:size: Size of created image in bytes (optional)
"""
base_dir = os.path.join(CONF.instances_path,
CONF.image_cache_subdirectory_name)
CONF.image_cache.subdirectory_name)
if not os.path.exists(base_dir):
fileutils.ensure_tree(base_dir)
base = os.path.join(base_dir, filename)

11
nova/virt/libvirt/imagecache.py
View File

@ -137,7 +137,7 @@ class ImageCacheManager(imagecache.ImageCacheManager):
if backing_file:
backing_path = os.path.join(
CONF.instances_path,
CONF.image_cache_subdirectory_name,
CONF.image_cache.subdirectory_name,
backing_file)
if backing_path not in inuse_images:
inuse_images.append(backing_path)
@ -236,15 +236,16 @@ class ImageCacheManager(imagecache.ImageCacheManager):
def _remove_swap_file(self, base_file):
"""Remove a single swap base file if it is old enough."""
maxage = CONF.remove_unused_original_minimum_age_seconds
maxage = CONF.image_cache.remove_unused_original_minimum_age_seconds
self._remove_old_enough_file(base_file, maxage, remove_lock=False)
def _remove_base_file(self, base_file):
"""Remove a single base file if it is old enough."""
maxage = CONF.libvirt.remove_unused_resized_minimum_age_seconds
maxage = CONF.image_cache.remove_unused_resized_minimum_age_seconds
if base_file in self.originals:
maxage = CONF.remove_unused_original_minimum_age_seconds
maxage = (
CONF.image_cache.remove_unused_original_minimum_age_seconds)
self._remove_old_enough_file(base_file, maxage)
@ -330,7 +331,7 @@ class ImageCacheManager(imagecache.ImageCacheManager):
# created, but may remain from previous versions.
base_dir = os.path.join(CONF.instances_path,
CONF.image_cache_subdirectory_name)
CONF.image_cache.subdirectory_name)
if not os.path.exists(base_dir):
LOG.debug('Skipping verification, no base directory at %s',
base_dir)

3
nova/virt/vmwareapi/imagecache.py
View File

@ -140,7 +140,8 @@ class ImageCacheManager(imagecache.ImageCacheManager):
def _age_cached_images(self, context, datastore, dc_info,
ds_path):
"""Ages cached images."""
age_seconds = CONF.remove_unused_original_minimum_age_seconds
age_seconds = (
CONF.image_cache.remove_unused_original_minimum_age_seconds)
unused_images = self.originals - self.used_images
ds_browser = self._get_ds_browser(datastore.ref)
for image in unused_images:

10
nova/virt/vmwareapi/vmops.py
View File

@ -152,14 +152,14 @@ class VMwareVMOps(object):
# Enable more than one compute node to run on the same host
if CONF.vmware.cache_prefix:
base_folder = '%s%s' % (CONF.vmware.cache_prefix,
CONF.image_cache_subdirectory_name)
CONF.image_cache.subdirectory_name)
# Ensure that the base folder is unique per compute node
elif CONF.remove_unused_base_images:
elif CONF.image_cache.remove_unused_base_images:
base_folder = '%s%s' % (CONF.my_ip,
CONF.image_cache_subdirectory_name)
CONF.image_cache.subdirectory_name)
else:
# Aging disable ensures backward compatibility
base_folder = CONF.image_cache_subdirectory_name
base_folder = CONF.image_cache.subdirectory_name
return base_folder
def _extend_virtual_disk(self, instance, requested_size, name, dc_ref):
@ -1855,7 +1855,7 @@ class VMwareVMOps(object):
self._set_machine_id(client_factory, instance, network_info)
def manage_image_cache(self, context, instances):
if not CONF.remove_unused_base_images:
if not CONF.image_cache.remove_unused_base_images:
LOG.debug("Image aging disabled. Aging will not be done.")
return

22
releasenotes/notes/image_cache-conf-opts-moved-e552e4a2d59e056e.yaml Normal file
View File

@ -0,0 +1,22 @@
---
deprecations:
- |
The following conf options have been moved to the ``[image_cache]`` group
and renamed accordingly. The old option paths are deprecated and will be
removed in a future release.
.. list-table::
:header-rows: 1
* - Deprecated Option
- New Option
* - ``[DEFAULT]image_cache_manager_interval``
- ``[image_cache]manager_interval``
* - ``[DEFAULT]image_cache_subdirectory_name``
- ``[image_cache]subdirectory_name``
* - ``[DEFAULT]remove_unused_base_images``
- ``[image_cache]remove_unused_base_images``
* - ``[DEFAULT]remove_unused_original_minimum_age_seconds``
- ``[image_cache]remove_unused_original_minimum_age_seconds``
* - ``[libvirt]remove_unused_resized_minimum_age_seconds``
- ``[image_cache]remove_unused_resized_minimum_age_seconds``