Fixes LP Bug#878411 - No docs for image cache
Adds documentation on managing the image cache Adds a simple CLI program to queue an image for caching. Change-Id: I2b4c9708c521fdcfaf1c6b8f07505e130923c4b7
This commit is contained in:
parent
8f122d954b
commit
2a5a0e9a35
bin
doc/source
etc
glance
@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
@ -21,7 +19,8 @@
|
||||
"""
|
||||
Glance Image Cache Pre-fetcher
|
||||
|
||||
This is meant to be run as a periodic task from cron.
|
||||
This is meant to be run from the command line after queueing
|
||||
images to be pretched.
|
||||
"""
|
||||
|
||||
import gettext
|
||||
@ -41,7 +40,6 @@ gettext.install('glance', unicode=1)
|
||||
|
||||
from glance import version
|
||||
from glance.common import config
|
||||
from glance.common import wsgi
|
||||
|
||||
|
||||
def create_options(parser):
|
||||
|
56
bin/glance-cache-queue-image
Normal file
56
bin/glance-cache-queue-image
Normal file
@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
CLI Utility to queue one or more images for prefetching into the image cache
|
||||
"""
|
||||
|
||||
import gettext
|
||||
import optparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
# If ../glance/__init__.py exists, add ../ to Python search path, so that
|
||||
# it will override what happens to be installed in /usr/(local/)lib/python...
|
||||
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
|
||||
os.pardir,
|
||||
os.pardir))
|
||||
if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
|
||||
sys.path.insert(0, possible_topdir)
|
||||
|
||||
gettext.install('glance', unicode=1)
|
||||
|
||||
from glance import version
|
||||
from glance.common import config
|
||||
|
||||
USAGE = """
|
||||
%%prog [OPTIONS] <IMAGE_ID1> [<IMAGE_ID2> ... <IMAGE_IDN>]
|
||||
"""
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
oparser = optparse.OptionParser(version='%%prog %s'
|
||||
% version.version_string(),
|
||||
usage=USAGE)
|
||||
(options, args) = config.parse_options(oparser)
|
||||
|
||||
try:
|
||||
conf, app = config.load_paste_app('glance-queue-image', options, args)
|
||||
app.run(args)
|
||||
except RuntimeError, e:
|
||||
sys.exit("ERROR: %s" % e)
|
122
doc/source/cache.rst
Normal file
122
doc/source/cache.rst
Normal file
@ -0,0 +1,122 @@
|
||||
..
|
||||
Copyright 2011 OpenStack, LLC
|
||||
All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
not use this file except in compliance with the License. You may obtain
|
||||
a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
License for the specific language governing permissions and limitations
|
||||
under the License.
|
||||
|
||||
The Glance Image Cache
|
||||
======================
|
||||
|
||||
The Glance API server may be configured to have an optional local image cache.
|
||||
A local image cache stores a copy of image files, essentially enabling multiple
|
||||
API servers to serve the same image file, resulting in an increase in
|
||||
scalability due to an increased number of endpoints serving an image file.
|
||||
|
||||
This local image cache is transparent to the end user -- in other words, the
|
||||
end user doesn't know that the Glance API is streaming an image file from
|
||||
its local cache or from the actual backend storage system.
|
||||
|
||||
Managing the Glance Image Cache
|
||||
-------------------------------
|
||||
|
||||
While image files are automatically placed in the image cache on successful
|
||||
requests to ``GET /images/<IMAGE_ID>``, the image cache is not automatically
|
||||
managed. Here, we describe the basics of how to manage the local image cache
|
||||
on Glance API servers and how to automate this cache management.
|
||||
|
||||
Controlling the Growth of the Image Cache
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The image cache has a configurable maximum size (the ``image_cache_max_size``
|
||||
configuration file option. However, when images are succesfully returned
|
||||
from a call to ``GET /images/<IMAGE_ID>``, the image cache automatically
|
||||
writes the image file to its cache, regardless of whether the resulting
|
||||
write would make the image cache's size exceed the value of
|
||||
``image_cache_max_size``. In order to keep the image cache at or below this
|
||||
maximum cache size, you need to run the ``glance-cache-pruner`` executable.
|
||||
|
||||
The recommended practice is to use ``cron`` to fire ``glance-cache-pruner``
|
||||
at a regular interval.
|
||||
|
||||
Cleaning the Image Cache
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Over time, the image cache can accumulate image files that are either in
|
||||
a stalled or invalid state. Stalled image files are the result of an image
|
||||
cache write failing to complete. Invalid image files are the result of an
|
||||
image file not being written properly to disk.
|
||||
|
||||
To remove these types of files, you run the ``glance-cache-cleaner``
|
||||
executable.
|
||||
|
||||
The recommended practice is to use ``cron`` to fire ``glance-cache-cleaner``
|
||||
at a semi-regular interval.
|
||||
|
||||
Prefetching Images into the Image Cache
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Some installations have base (sometimes called "golden") images that are
|
||||
very commonly used to boot virtual machines. When spinning up a new API
|
||||
server, administrators may wish to prefetch these image files into the
|
||||
local image cache to ensure that reads of those popular image files come
|
||||
from a local cache.
|
||||
|
||||
To queue an image for prefetching, you can use one of the following methods:
|
||||
|
||||
* If the ``cache_manage`` middleware is enabled in the application pipeline,
|
||||
you may call ``PUT /cached-images/<IMAGE_ID>`` to queue the image with
|
||||
identifier ``<IMAGE_ID>``
|
||||
|
||||
* You may use the ``glance-cache-queue-image`` executable, supplying a list
|
||||
of image identifiers to queue for prefetching into the cache.
|
||||
|
||||
Example usage::
|
||||
|
||||
$> glance-cache-queue-image 12345 ABCDE
|
||||
|
||||
would queue the images with identifiers ``12345`` and ``ABCDE`` for
|
||||
prefetching.
|
||||
|
||||
Once you have queued the images you wish to prefetch, call the
|
||||
``glance-cache-prefetch`` executable, which will prefetch all queued images
|
||||
concurrently, reporting the results of the fetch for each image, as shown
|
||||
below::
|
||||
|
||||
TODO
|
||||
|
||||
Finding Which Images are in the Image Cache
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
You can find out which images are in the image cache using one of the
|
||||
following methods:
|
||||
|
||||
* If the ``cache_manage`` middleware is enabled in the application pipeline,
|
||||
you may call ``GET /cached-images`` to see a JSON-serialized list of
|
||||
mappings that show cached images, the number of cache hits on each image,
|
||||
the size of the image, and the times they were last accessed.
|
||||
|
||||
* You can issue the following call on \*nix systems::
|
||||
|
||||
$> ls -lhR $IMAGE_CACHE_DIR
|
||||
|
||||
where ``$IMAGE_CACHE_DIR`` is the value of the ``image_cache_dir``
|
||||
configuration variable.
|
||||
|
||||
Note that the image's cache hit is not shown using this method.
|
||||
|
||||
Manually Removing Images from the Image Cache
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
If the ``cache_manage`` middleware is enabled, you may call
|
||||
``DELETE /cached-images/<IMAGE_ID>`` to remove the image file for image
|
||||
with identifier ``<IMAGE_ID>`` from the cache.
|
@ -549,6 +549,20 @@ When using the ``sqlite`` cache driver, you can set the name of the database
|
||||
that will be used to store the cached images information. The database
|
||||
is always contained in the ``image_cache_dir``.
|
||||
|
||||
* ``image_cache_max_size=SIZE``
|
||||
|
||||
Optional.
|
||||
|
||||
Default: ``10737418240`` (10 GB)
|
||||
|
||||
Size, in bytes, that the image cache should be constrained to. Images files
|
||||
are cached automatically in the local image cache, even if the writing of that
|
||||
image file would put the total cache size over this size. The
|
||||
``glance-cache-pruner`` executable is what prunes the image cache to be equal
|
||||
to or less than this value. The ``glance-cache-pruner`` executable is designed
|
||||
to be run via cron on a regular basis. See more about this executable in
|
||||
`Controlling the Growth of the Image Cache`
|
||||
|
||||
Configuring the Glance Registry
|
||||
-------------------------------
|
||||
|
||||
|
@ -66,6 +66,7 @@ Using Glance
|
||||
glanceapi
|
||||
client
|
||||
authentication
|
||||
cache
|
||||
|
||||
Developer Docs
|
||||
==============
|
||||
|
@ -28,7 +28,7 @@ image_cache_stall_time = 86400
|
||||
image_cache_invalid_entry_grace_period = 3600
|
||||
|
||||
# Max cache size in bytes
|
||||
image_cache_max_size = 1073741824
|
||||
image_cache_max_size = 10737418240
|
||||
|
||||
# Address to find the registry server
|
||||
registry_host = 0.0.0.0
|
||||
@ -44,3 +44,6 @@ paste.app_factory = glance.image_cache.prefetcher:app_factory
|
||||
|
||||
[app:glance-cleaner]
|
||||
paste.app_factory = glance.image_cache.cleaner:app_factory
|
||||
|
||||
[app:glance-queue-image]
|
||||
paste.app_factory = glance.image_cache.queue_image:app_factory
|
||||
|
@ -78,7 +78,7 @@ class Prefetcher(object):
|
||||
"images in queue."))
|
||||
return False
|
||||
|
||||
logger.info(_("Successfully cache all %d images"), num_images)
|
||||
logger.info(_("Successfully cached all %d images"), num_images)
|
||||
return True
|
||||
|
||||
|
||||
|
84
glance/image_cache/queue_image.py
Normal file
84
glance/image_cache/queue_image.py
Normal file
@ -0,0 +1,84 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Queues images for prefetching into the image cache
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import eventlet
|
||||
|
||||
from glance.common import config
|
||||
from glance.common import context
|
||||
from glance.common import exception
|
||||
from glance.image_cache import ImageCache
|
||||
from glance import registry
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Queuer(object):
|
||||
|
||||
def __init__(self, options):
|
||||
self.options = options
|
||||
self.cache = ImageCache(options)
|
||||
registry.configure_registry_client(options)
|
||||
|
||||
def queue_image(self, image_id):
|
||||
ctx = context.RequestContext(is_admin=True, show_deleted=True)
|
||||
try:
|
||||
image_meta = registry.get_image_metadata(ctx, image_id)
|
||||
if image_meta['status'] != 'active':
|
||||
logger.warn(_("Image '%s' is not active. Not queueing."),
|
||||
image_id)
|
||||
return False
|
||||
|
||||
except exception.NotFound:
|
||||
logger.warn(_("No metadata found for image '%s'"), image_id)
|
||||
return False
|
||||
|
||||
logger.debug(_("Queueing image '%s'"), image_id)
|
||||
self.cache.queue_image(image_id)
|
||||
return True
|
||||
|
||||
def run(self, images):
|
||||
|
||||
num_images = len(images)
|
||||
if num_images == 0:
|
||||
logger.debug(_("No images to queue!"))
|
||||
return True
|
||||
|
||||
logger.debug(_("Received %d images to queue"), num_images)
|
||||
|
||||
pool = eventlet.GreenPool(num_images)
|
||||
results = pool.imap(self.queue_image, images)
|
||||
successes = sum([1 for r in results if r is True])
|
||||
if successes != num_images:
|
||||
logger.error(_("Failed to successfully queue all "
|
||||
"images in queue."))
|
||||
return False
|
||||
|
||||
logger.info(_("Successfully queued all %d images"), num_images)
|
||||
return True
|
||||
|
||||
|
||||
def app_factory(global_config, **local_conf):
|
||||
conf = global_config.copy()
|
||||
conf.update(local_conf)
|
||||
return Prefetcher(conf)
|
@ -24,9 +24,7 @@ import unittest
|
||||
import stubout
|
||||
|
||||
from glance import image_cache
|
||||
from glance.image_cache import prefetcher
|
||||
from glance.common import exception
|
||||
from glance.tests import stubs
|
||||
from glance.tests.utils import skip_if_disabled
|
||||
|
||||
FIXTURE_DATA = '*' * 1024
|
||||
@ -204,25 +202,6 @@ class ImageCacheTestCase(object):
|
||||
self.assertEqual(self.cache.get_cache_queue(),
|
||||
['0', '1', '2'])
|
||||
|
||||
@skip_if_disabled
|
||||
def test_prefetcher(self):
|
||||
"""
|
||||
Test that the prefetcher application works
|
||||
"""
|
||||
stubs.stub_out_registry_server(self.stubs)
|
||||
FIXTURE_FILE = StringIO.StringIO(FIXTURE_DATA)
|
||||
|
||||
# Should return True since there is nothing in the queue
|
||||
pf = prefetcher.Prefetcher(self.options)
|
||||
self.assertTrue(pf.run())
|
||||
|
||||
for x in xrange(2, 3):
|
||||
self.assertTrue(self.cache.queue_image(x))
|
||||
|
||||
# Should return False since there is no metadata for these
|
||||
# images in the registry
|
||||
self.assertFalse(pf.run())
|
||||
|
||||
|
||||
class TestImageCacheXattr(unittest.TestCase,
|
||||
ImageCacheTestCase):
|
||||
@ -257,12 +236,10 @@ class TestImageCacheXattr(unittest.TestCase,
|
||||
'registry_host': '0.0.0.0',
|
||||
'registry_port': 9191}
|
||||
self.cache = image_cache.ImageCache(self.options)
|
||||
self.stubs = stubout.StubOutForTesting()
|
||||
|
||||
def tearDown(self):
|
||||
if os.path.exists(self.cache_dir):
|
||||
shutil.rmtree(self.cache_dir)
|
||||
self.stubs.UnsetAll()
|
||||
|
||||
|
||||
class TestImageCacheSqlite(unittest.TestCase,
|
||||
@ -297,9 +274,7 @@ class TestImageCacheSqlite(unittest.TestCase,
|
||||
'registry_host': '0.0.0.0',
|
||||
'registry_port': 9191}
|
||||
self.cache = image_cache.ImageCache(self.options)
|
||||
self.stubs = stubout.StubOutForTesting()
|
||||
|
||||
def tearDown(self):
|
||||
if os.path.exists(self.cache_dir):
|
||||
shutil.rmtree(self.cache_dir)
|
||||
self.stubs.UnsetAll()
|
||||
|
Loading…
x
Reference in New Issue
Block a user