openstack/glance
Code Issues Proposed changes

Create bin/glance-pruner

Browse Source
This commit is contained in:
Rick Harris
2011-07-12 02:55:45 -05:00
parent 4d4ef5482a
commit a4369c75b3
6 changed files with 201 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
bin/glance-pruner Executable file
View File

@@ -0,0 +1,65 @@
#!/usr/bin/env python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Glance Image Cache Pruner
This is meant to be run as a periodic task, perhaps every half-hour.
"""
import optparse
import os
import sys
# If ../glance/__init__.py exists, add ../ to Python search path, so that
# it will override what happens to be installed in /usr/(local/)lib/python...
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir,
os.pardir))
if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
sys.path.insert(0, possible_topdir)
from glance import version
from glance.common import config
from glance.common import wsgi
def create_options(parser):
"""
Sets up the CLI and config-file options that may be
parsed and program commands.
:param parser: The option parser
"""
config.add_common_options(parser)
config.add_log_options(parser)
if __name__ == '__main__':
oparser = optparse.OptionParser(version='%%prog %s'
% version.version_string())
create_options(oparser)
(options, args) = config.parse_options(oparser)
try:
conf, app = config.load_paste_app('glance-pruner', options, args)
app.run()
except RuntimeError, e:
sys.exit("ERROR: %s" % e)

14
etc/glance-api.conf
View File

@@ -56,20 +56,8 @@ swift_store_create_container_on_put = False
# Whether to enable the caching of image data
image_cache_enabled = False
image_cache_max_size_bytes = 1073741824
# Percentage of the cache that should be freed (in addition to the overage)
# when the cache is pruned
#
# A percentage of 0% means we prune only as many files as needed to remain
# under the cache's max_size. This is space efficient but will lead to
# constant pruning as the size bounces just-above and just-below the max_size.
#
# To mitigate this 'thrashing', you can specify an additional amount of the
# cache that should be tossed out on each prune.
image_cache_percent_extra_to_free = 5
# Directory that the Image Cache writes data to
# Make sure this is also set in glance-pruner.conf
image_cache_datadir = /var/lib/glance/image-cache/
[pipeline:glance-api]

28
etc/glance-pruner.conf Normal file
View File

@@ -0,0 +1,28 @@
[DEFAULT]
# Show more verbose log output (sets INFO log level output)
verbose = True
# Show debugging output in logs (sets DEBUG log level output)
debug = False
log_file = /var/log/glance/pruner.log
image_cache_max_size_bytes = 1073741824
# Percentage of the cache that should be freed (in addition to the overage)
# when the cache is pruned
#
# A percentage of 0% means we prune only as many files as needed to remain
# under the cache's max_size. This is space efficient but will lead to
# constant pruning as the size bounces just-above and just-below the max_size.
#
# To mitigate this 'thrashing', you can specify an additional amount of the
# cache that should be tossed out on each prune.
image_cache_percent_extra_to_free = 0.20
# Directory that the Image Cache writes data to
# Make sure this is also set in glance-api.conf
image_cache_datadir = /var/lib/glance/image-cache/
[app:glance-pruner]
paste.app_factory = glance.image_cache.pruner:app_factory

4
glance/api/v1/images.py
View File

@@ -29,7 +29,7 @@ from webob.exc import (HTTPNotFound,
HTTPConflict,
HTTPBadRequest)
from glance.api import image_cache
from glance import image_cache
from glance.common import exception
from glance.common import wsgi
from glance.store import (get_from_backend,
@@ -206,8 +206,6 @@ class Controller(object):
for chunk in chunks:
cache_file.write(chunk)
yield chunk
#TODO(sirp): call this from cron
cache.prune()
cache = image_cache.ImageCache(self.options)
if cache.enabled:

71
glance/image_cache/__init__.py Normal file
View File

@@ -0,0 +1,71 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
LRU Cache for Image Data
"""
from contextlib import contextmanager
import logging
import os
from glance.common import config
logger = logging.getLogger('glance.image_cache')
class ImageCache(object):
def __init__(self, options):
self.options = options
self._make_cache_directory_if_needed()
def _make_cache_directory_if_needed(self):
if self.enabled and not os.path.exists(self.path):
logger.info("image cache directory doesn't exist, creating '%s'",
self.path)
os.makedirs(self.path)
@property
def enabled(self):
return config.get_option(
self.options, 'image_cache_enabled', type='bool', default=False)
@property
def path(self):
"""This is the base path for the image cache"""
datadir = self.options['image_cache_datadir']
return datadir
def path_for_image(self, image_meta):
"""This crafts an absolute path to a specific entry"""
image_id = image_meta['id']
return os.path.join(self.path, str(image_id))
@contextmanager
def open(self, image_meta, mode="r"):
path = self.path_for_image(image_meta)
with open(path, mode) as cache_file:
yield cache_file
def hit(self, image_meta):
path = self.path_for_image(image_meta)
return os.path.exists(path)
def delete(self, image_meta):
path = self.path_for_image(image_meta)
logger.debug("deleting image cache entry '%s'", path)
if os.path.exists(path):
os.unlink(path)

82
glance/api/image_cache.py → glance/image_cache/pruner.py
View File

@@ -1,30 +1,37 @@
from contextlib import contextmanager
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Prunes the Image Cache
"""
import logging
import os
import stat
import types
from glance.common import config
from glance import utils
from glance.image_cache import ImageCache
logger = logging.getLogger('glance.api.image_cache')
logger = logging.getLogger('glance.image_cache.pruner')
class ImageCache(object):
class Pruner(object):
def __init__(self, options):
self.options = options
self._make_cache_directory_if_needed()
def _make_cache_directory_if_needed(self):
if self.enabled and not os.path.exists(self.path):
logger.info("image cache directory doesn't exist, creating '%s'",
self.path)
os.makedirs(self.path)
@property
def enabled(self):
return config.get_option(
self.options, 'image_cache_enabled', type='bool', default=False)
self.cache = ImageCache(options)
@property
def max_size(self):
@@ -33,40 +40,14 @@ class ImageCache(object):
self.options, 'image_cache_max_size_bytes',
type='int', default=default)
@property
def path(self):
"""This is the base path for the image cache"""
datadir = self.options['image_cache_datadir']
return datadir
def path_for_image(self, image_meta):
"""This crafts an absolute path to a specific entry"""
image_id = image_meta['id']
return os.path.join(self.path, str(image_id))
@contextmanager
def open(self, image_meta, mode="r"):
path = self.path_for_image(image_meta)
with open(path, mode) as cache_file:
yield cache_file
def hit(self, image_meta):
path = self.path_for_image(image_meta)
return os.path.exists(path)
def delete(self, image_meta):
path = self.path_for_image(image_meta)
logger.debug("deleting image cache entry '%s'", path)
if os.path.exists(path):
os.unlink(path)
@property
def percent_extra_to_free(self):
return config.get_option(
self.options, 'image_cache_percent_extra_to_free',
type='float', default=0.05)
def prune(self):
def run(self):
"""Prune the cache using an LRU strategy"""
# NOTE(sirp): 'Recency' is determined via the filesystem, first using
@@ -80,9 +61,10 @@ class ImageCache(object):
# times elsewhere (either as a separate file, in the DB, or as
# an xattr).
def get_stats():
cache_path = self.cache.path
stats = []
for fname in os.listdir(self.path):
path = os.path.join(self.path, fname)
for fname in os.listdir(cache_path):
path = os.path.join(cache_path, fname)
file_info = os.stat(path)
mode = file_info[stat.ST_MODE]
if not stat.S_ISREG(mode):
@@ -127,3 +109,9 @@ class ImageCache(object):
freed = prune_lru(stats, to_free)
logger.debug("finished pruning, freed %(freed)d bytes" % locals())
def app_factory(global_config, **local_conf):
conf = global_config.copy()
conf.update(local_conf)
return Pruner(conf)