From ad9e9ca3f741af714b9faf46224bf12e4a2693bd Mon Sep 17 00:00:00 2001 From: Jay Pipes Date: Tue, 18 Oct 2011 09:27:27 -0400 Subject: [PATCH] Overhauls the image cache to be truly optional Fixes LP Bug#874580 - keyerror 'location' when fetch errors Fixes LP Bug#817570 - Make new image cache a true extension Fixes LP Bug#872372 - Image cache has virtually no unit test coverage * Adds unit tests for the image cache (coverage goes from 26% to 100%) * Removes caching logic from the images controller and places it into a removeable transparent caching middleware * Adds a functional test case that verifies caching of an image and subsequent cache hits * Removes the image_cache_enabled configuration variable, since it's now enabled by simply including the cache in the application pipeline * Adds a singular glance-cache.conf to etc/ that replaces the multiple glance-pruner.conf, glance-reaper.conf and glance-prefetcher.conf files * Adds documentation on enabling and configuring the image cache TODO: Add documentation on the image cache utilities, like reaper, prefetcher, etc. Change-Id: I58845871deee26f81ffabe1750adc472ce5b3797 --- doc/source/configuring.rst | 48 ++++ etc/glance-api.conf | 35 +-- etc/glance-cache.conf | 56 +++++ glance/api/middleware/cache.py | 180 ++++++++++++++ .../{image_cache.py => cache_manage.py} | 6 +- glance/api/v1/images.py | 59 +---- glance/common/utils.py | 15 ++ glance/image_cache/__init__.py | 44 ++-- glance/tests/functional/__init__.py | 9 +- glance/tests/functional/test_image_cache.py | 94 +++++++ glance/tests/stubs.py | 62 +++++ glance/tests/unit/test_cache_middleware.py | 114 +++++++++ glance/tests/unit/test_image_cache.py | 233 ++++++++++++++++-- glance/utils.py | 14 -- 14 files changed, 833 insertions(+), 136 deletions(-) create mode 100644 etc/glance-cache.conf create mode 100644 glance/api/middleware/cache.py rename glance/api/middleware/{image_cache.py => cache_manage.py} (91%) create mode 100644 glance/tests/functional/test_image_cache.py create mode 100644 glance/tests/unit/test_cache_middleware.py diff --git a/doc/source/configuring.rst b/doc/source/configuring.rst index ef6262155a..5977aad769 100644 --- a/doc/source/configuring.rst +++ b/doc/source/configuring.rst @@ -468,6 +468,54 @@ To set up a user named ``glance`` with minimal permissions, using a pool called ceph-authtool --gen-key --name client.glance --cap mon 'allow r' --cap osd 'allow rwx pool=images' /etc/glance/rbd.keyring ceph auth add client.glance -i /etc/glance/rbd.keyring +Configuring the Image Cache +--------------------------- + +Glance API servers can be configured to have a local image cache. Caching of +image files is transparent and happens using a piece of middleware that can +optionally be placed in the server application pipeline. + +Enabling the Image Cache Middleware +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To enable the image cache middleware, you would insert the cache middleware +into your application pipeline **after** the appropriate context middleware. + +The cache middleware should be in your ``glance-api.conf`` in a section titled +``[filter:cache]``. It should look like this:: + + [filter:cache] + paste.filter_factory = glance.api.middleware.cache:filter_factory + + +For example, suppose your application pipeline in the ``glance-api.conf`` file +looked like so:: + + [pipeline:glance-api] + pipeline = versionnegotiation context apiv1app + +In the above application pipeline, you would add the cache middleware after the +context middleware, like so:: + + [pipeline:glance-api] + pipeline = versionnegotiation context cache apiv1app + +And that would give you a transparent image cache on the API server. + +Configuration Options Affecting the Image Cache +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +One main configuration file option affects the image cache. + + * ``image_cache_datadir=PATH`` + +Required when image cache middleware is enabled. + +Default: ``/var/lib/glance/image-cache`` + +This is the root directory where the image cache will write its +cached image files. Make sure the directory is writeable by the +user running the ``glance-api`` server Configuring the Glance Registry ------------------------------- diff --git a/etc/glance-api.conf b/etc/glance-api.conf index f7598ed044..aa58875196 100644 --- a/etc/glance-api.conf +++ b/etc/glance-api.conf @@ -164,18 +164,6 @@ rbd_store_pool = images # For best performance, this should be a power of two rbd_store_chunk_size = 8 -# ============ Image Cache Options ======================== - -image_cache_enabled = False - -# Directory that the Image Cache writes data to -# Make sure this is also set in glance-pruner.conf -image_cache_datadir = /var/lib/glance/image-cache/ - -# Number of seconds after which we should consider an incomplete image to be -# stalled and eligible for reaping -image_cache_stall_timeout = 86400 - # ============ Delayed Delete Options ============================= # Turn on/off delayed delete @@ -188,15 +176,25 @@ scrub_time = 43200 # Make sure this is also set in glance-scrubber.conf scrubber_datadir = /var/lib/glance/scrubber +# =============== Image Cache Options ============================= + +# Directory that the Image Cache writes data to +image_cache_datadir = /var/lib/glance/image-cache/ + [pipeline:glance-api] pipeline = versionnegotiation context apiv1app # NOTE: use the following pipeline for keystone # pipeline = versionnegotiation authtoken auth-context apiv1app -# To enable Image Cache Management API replace pipeline with below: -# pipeline = versionnegotiation context imagecache apiv1app +# To enable transparent caching of image files replace pipeline with below: +# pipeline = versionnegotiation context cache apiv1app # NOTE: use the following pipeline for keystone auth (with caching) -# pipeline = versionnegotiation authtoken auth-context imagecache apiv1app +# pipeline = versionnegotiation authtoken auth-context cache apiv1app + +# To enable Image Cache Management API replace pipeline with below: +# pipeline = versionnegotiation context cachemanage apiv1app +# NOTE: use the following pipeline for keystone auth (with caching) +# pipeline = versionnegotiation authtoken auth-context cachemanage apiv1app [pipeline:versions] pipeline = versionsapp @@ -210,8 +208,11 @@ paste.app_factory = glance.api.v1:app_factory [filter:versionnegotiation] paste.filter_factory = glance.api.middleware.version_negotiation:filter_factory -[filter:imagecache] -paste.filter_factory = glance.api.middleware.image_cache:filter_factory +[filter:cache] +paste.filter_factory = glance.api.middleware.cache:filter_factory + +[filter:cachemanage] +paste.filter_factory = glance.api.middleware.cache_manage:filter_factory [filter:context] paste.filter_factory = glance.common.context:filter_factory diff --git a/etc/glance-cache.conf b/etc/glance-cache.conf new file mode 100644 index 0000000000..53b01e3b2e --- /dev/null +++ b/etc/glance-cache.conf @@ -0,0 +1,56 @@ +[DEFAULT] +# Show more verbose log output (sets INFO log level output) +verbose = True + +# Show debugging output in logs (sets DEBUG log level output) +debug = False + +log_file = /var/log/glance/image-cache.log + +# Send logs to syslog (/dev/log) instead of to file specified by `log_file` +use_syslog = False + +# Directory that the Image Cache writes data to +image_cache_datadir = /var/lib/glance/image-cache/ + +# Number of seconds after which we should consider an incomplete image to be +# stalled and eligible for reaping +image_cache_stall_timeout = 86400 + +# image_cache_invalid_entry_grace_period - seconds +# +# If an exception is raised as we're writing to the cache, the cache-entry is +# deemed invalid and moved to /invalid so that it can be +# inspected for debugging purposes. +# +# This is number of seconds to leave these invalid images around before they +# are elibible to be reaped. +image_cache_invalid_entry_grace_period = 3600 + +image_cache_max_size_bytes = 1073741824 + +# Percentage of the cache that should be freed (in addition to the overage) +# when the cache is pruned +# +# A percentage of 0% means we prune only as many files as needed to remain +# under the cache's max_size. This is space efficient but will lead to +# constant pruning as the size bounces just-above and just-below the max_size. +# +# To mitigate this 'thrashing', you can specify an additional amount of the +# cache that should be tossed out on each prune. +image_cache_percent_extra_to_free = 0.20 + +# Address to find the registry server +registry_host = 0.0.0.0 + +# Port the registry server is listening on +registry_port = 9191 + +[app:glance-pruner] +paste.app_factory = glance.image_cache.pruner:app_factory + +[app:glance-prefetcher] +paste.app_factory = glance.image_cache.prefetcher:app_factory + +[app:glance-reaper] +paste.app_factory = glance.image_cache.reaper:app_factory diff --git a/glance/api/middleware/cache.py b/glance/api/middleware/cache.py new file mode 100644 index 0000000000..8292c39f86 --- /dev/null +++ b/glance/api/middleware/cache.py @@ -0,0 +1,180 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Transparent image file caching middleware, designed to live on +Glance API nodes. When images are requested from the API node, +this middleware caches the returned image file to local filesystem. + +When subsequent requests for the same image file are received, +the local cached copy of the image file is returned. +""" + +import httplib +import logging +import re +import shutil + +from glance import image_cache +from glance import registry +from glance.api.v1 import images +from glance.common import exception +from glance.common import utils +from glance.common import wsgi + +import webob + +logger = logging.getLogger(__name__) +get_images_re = re.compile(r'^(/v\d+)*/images/(.+)$') + + +class CacheFilter(wsgi.Middleware): + + def __init__(self, app, options): + self.options = options + self.cache = image_cache.ImageCache(options) + self.serializer = images.ImageSerializer() + logger.info(_("Initialized image cache middleware using datadir: %s"), + options.get('image_cache_datadir')) + super(CacheFilter, self).__init__(app) + + def process_request(self, request): + """ + For requests for an image file, we check the local image + cache. If present, we return the image file, appending + the image metadata in headers. If not present, we pass + the request on to the next application in the pipeline. + """ + if request.method != 'GET': + return None + + match = get_images_re.match(request.path) + if not match: + return None + + image_id = match.group(2) + if self.cache.hit(image_id): + logger.debug(_("Cache hit for image '%s'"), image_id) + image_iterator = self.get_from_cache(image_id) + context = request.context + try: + image_meta = registry.get_image_metadata(context, image_id) + + response = webob.Response() + return self.serializer.show(response, { + 'image_iterator': image_iterator, + 'image_meta': image_meta}) + except exception.NotFound: + msg = _("Image cache contained image file for image '%s', " + "however the registry did not contain metadata for " + "that image!" % image_id) + logger.error(msg) + return None + + # Make sure we're not already prefetching or caching the image + # that just generated the miss + if self.cache.is_image_currently_prefetching(image_id): + logger.debug(_("Image '%s' is already being prefetched," + " not tee'ing into the cache"), image_id) + return None + elif self.cache.is_image_currently_being_written(image_id): + logger.debug(_("Image '%s' is already being cached," + " not tee'ing into the cache"), image_id) + return None + + # NOTE(sirp): If we're about to download and cache an + # image which is currently in the prefetch queue, just + # delete the queue items since we're caching it anyway + if self.cache.is_image_queued_for_prefetch(image_id): + self.cache.delete_queued_prefetch_image(image_id) + return None + + def process_response(self, resp): + """ + We intercept the response coming back from the main + images Resource, caching image files to the cache + """ + if not self.get_status_code(resp) == httplib.OK: + return resp + + request = resp.request + if request.method != 'GET': + return resp + + match = get_images_re.match(request.path) + if match is None: + return resp + + image_id = match.group(2) + if not self.cache.hit(image_id): + # Make sure we're not already prefetching or caching the image + # that just generated the miss + if self.cache.is_image_currently_prefetching(image_id): + logger.debug(_("Image '%s' is already being prefetched," + " not tee'ing into the cache"), image_id) + return resp + if self.cache.is_image_currently_being_written(image_id): + logger.debug(_("Image '%s' is already being cached," + " not tee'ing into the cache"), image_id) + return resp + + logger.debug(_("Tee'ing image '%s' into cache"), image_id) + # TODO(jaypipes): This is so incredibly wasteful, but because + # the image cache needs the image's name, we have to do this. + # In the next iteration, remove the image cache's need for + # any attribute other than the id... + image_meta = registry.get_image_metadata(request.context, + image_id) + resp.app_iter = self.get_from_store_tee_into_cache( + image_meta, resp.app_iter) + return resp + + def get_status_code(self, response): + """ + Returns the integer status code from the response, which + can be either a Webob.Response (used in testing) or httplib.Response + """ + if hasattr(response, 'status_int'): + return response.status_int + return response.status + + def get_from_store_tee_into_cache(self, image_meta, image_iterator): + """Called if cache miss""" + with self.cache.open(image_meta, "wb") as cache_file: + for chunk in image_iterator: + cache_file.write(chunk) + yield chunk + + def get_from_cache(self, image_id): + """Called if cache hit""" + with self.cache.open_for_read(image_id) as cache_file: + chunks = utils.chunkiter(cache_file) + for chunk in chunks: + yield chunk + + +def filter_factory(global_conf, **local_conf): + """ + Factory method for paste.deploy + """ + conf = global_conf.copy() + conf.update(local_conf) + + def filter(app): + return CacheFilter(app, conf) + + return filter diff --git a/glance/api/middleware/image_cache.py b/glance/api/middleware/cache_manage.py similarity index 91% rename from glance/api/middleware/image_cache.py rename to glance/api/middleware/cache_manage.py index 72a5f3e97b..935f512cc7 100644 --- a/glance/api/middleware/image_cache.py +++ b/glance/api/middleware/cache_manage.py @@ -27,9 +27,9 @@ from glance.common import wsgi logger = logging.getLogger('glance.api.middleware.image_cache') -class ImageCacheFilter(wsgi.Middleware): +class CacheManageFilter(wsgi.Middleware): def __init__(self, app, options): - super(ImageCacheFilter, self).__init__(app) + super(CacheManageFilter, self).__init__(app) map = app.map resource = cached_images.create_resource(options) @@ -52,6 +52,6 @@ def filter_factory(global_conf, **local_conf): conf.update(local_conf) def filter(app): - return ImageCacheFilter(app, conf) + return CacheManageFilter(app, conf) return filter diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index b390ecc3ad..c468531649 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -207,10 +207,9 @@ class Controller(api.BaseController): :raises HTTPNotFound if image is not available to user """ - image = self.get_active_image_meta_or_404(req, id) + image_meta = self.get_active_image_meta_or_404(req, id) def get_from_store(image_meta): - """Called if caching disabled""" try: location = image_meta['location'] image_data, image_size = get_from_backend(location) @@ -219,61 +218,11 @@ class Controller(api.BaseController): raise HTTPNotFound(explanation="%s" % e) return image_data - def get_from_cache(image, cache): - """Called if cache hit""" - with cache.open(image, "rb") as cache_file: - chunks = utils.chunkiter(cache_file) - for chunk in chunks: - yield chunk - - def get_from_store_tee_into_cache(image, cache): - """Called if cache miss""" - with cache.open(image, "wb") as cache_file: - chunks = get_from_store(image) - for chunk in chunks: - cache_file.write(chunk) - yield chunk - - cache = image_cache.ImageCache(self.options) - if cache.enabled: - if cache.hit(id): - # hit - logger.debug(_("image '%s' is a cache HIT"), id) - image_iterator = get_from_cache(image, cache) - else: - # miss - logger.debug(_("image '%s' is a cache MISS"), id) - - # Make sure we're not already prefetching or caching the image - # that just generated the miss - if cache.is_image_currently_prefetching(id): - logger.debug(_("image '%s' is already being prefetched," - " not tee'ing into the cache"), id) - image_iterator = get_from_store(image) - elif cache.is_image_currently_being_written(id): - logger.debug(_("image '%s' is already being cached," - " not tee'ing into the cache"), id) - image_iterator = get_from_store(image) - else: - # NOTE(sirp): If we're about to download and cache an - # image which is currently in the prefetch queue, just - # delete the queue items since we're caching it anyway - if cache.is_image_queued_for_prefetch(id): - cache.delete_queued_prefetch_image(id) - - logger.debug(_("tee'ing image '%s' into cache"), id) - image_iterator = get_from_store_tee_into_cache( - image, cache) - else: - # disabled - logger.debug(_("image cache DISABLED, retrieving image '%s'" - " from store"), id) - image_iterator = get_from_store(image) - - del image['location'] + image_iterator = get_from_store(image_meta) + del image_meta['location'] return { 'image_iterator': image_iterator, - 'image_meta': image, + 'image_meta': image_meta, } def _reserve(self, req, image_meta): diff --git a/glance/common/utils.py b/glance/common/utils.py index 0655db6d94..0e7f7f2b9d 100644 --- a/glance/common/utils.py +++ b/glance/common/utils.py @@ -35,6 +35,21 @@ from glance.common import exception TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +def chunkiter(fp, chunk_size=65536): + """ + Return an iterator to a file-like obj which yields fixed size chunks + + :param fp: a file-like object + :param chunk_size: maximum size of chunk + """ + while True: + chunk = fp.read(chunk_size) + if chunk: + yield chunk + else: + break + + def bool_from_string(subject): """ Interpret a string as a boolean. diff --git a/glance/image_cache/__init__.py b/glance/image_cache/__init__.py index 4b20073a5a..463123d9dc 100644 --- a/glance/image_cache/__init__.py +++ b/glance/image_cache/__init__.py @@ -18,6 +18,7 @@ """ LRU Cache for Image Data """ + from contextlib import contextmanager import datetime import itertools @@ -28,18 +29,15 @@ import time from glance.common import config from glance.common import exception +from glance.common import utils as cutils from glance import utils -logger = logging.getLogger('glance.image_cache') +logger = logging.getLogger(__name__) class ImageCache(object): - """Provides an LRU cache for image data. - - Data is cached on READ not on WRITE; meaning if the cache is enabled, we - attempt to read from the cache first, if we don't find the data, we begin - streaming the data from the 'store' while simultaneously tee'ing the data - into the cache. Subsequent reads will generate cache HITs for this image. + """ + Provides an LRU cache for image data. Assumptions =========== @@ -81,8 +79,6 @@ class ImageCache(object): def _make_cache_directory_if_needed(self): """Creates main cache directory along with incomplete subdirectory""" - if not self.enabled: - return # NOTE(sirp): making the incomplete_path will have the effect of # creating the main cache path directory as well @@ -90,16 +86,7 @@ class ImageCache(object): self.prefetching_path] for path in paths: - if os.path.exists(path): - continue - logger.info(_("image cache directory doesn't exist, " - "creating '%s'"), path) - os.makedirs(path) - - @property - def enabled(self): - return config.get_option( - self.options, 'image_cache_enabled', type='bool', default=False) + cutils.safe_mkdirs(path) @property def path(self): @@ -222,6 +209,23 @@ class ImageCache(object): else: commit() + @contextmanager + def open_for_read(self, image_id): + path = self.path_for_image(image_id) + with open(path, 'rb') as cache_file: + yield cache_file + + utils.inc_xattr(path, 'hits') # bump the hit count + + def get_hit_count(self, image_id): + """ + Return the number of hits that an image has + + :param image_id: Opaque image identifier + """ + path = self.path_for_image(image_id) + return int(utils.get_xattr(path, 'hits', default=0)) + @contextmanager def _open_read(self, image_meta, mode): image_id = image_meta['id'] @@ -390,7 +394,7 @@ class ImageCache(object): yield entry def incomplete_entries(self): - """Cache info for invalid cached images""" + """Cache info for incomplete cached images""" for entry in self._base_entries(self.incomplete_path): yield entry diff --git a/glance/tests/functional/__init__.py b/glance/tests/functional/__init__.py index 3ddb2580cc..3cd7a5cb8c 100644 --- a/glance/tests/functional/__init__.py +++ b/glance/tests/functional/__init__.py @@ -148,6 +148,8 @@ class ApiServer(Server): self.default_store = 'file' self.key_file = "" self.cert_file = "" + self.image_cache_datadir = os.path.join(self.test_dir, + 'cache') self.image_dir = os.path.join(self.test_dir, "images") self.pid_file = os.path.join(self.test_dir, @@ -172,6 +174,7 @@ class ApiServer(Server): self.rbd_store_chunk_size = 4 self.delayed_delete = delayed_delete self.owner_is_tenant = True + self.cache_pipeline = "" # Set to cache for cache middleware self.conf_base = """[DEFAULT] verbose = %(verbose)s debug = %(debug)s @@ -202,9 +205,10 @@ delayed_delete = %(delayed_delete)s owner_is_tenant = %(owner_is_tenant)s scrub_time = 5 scrubber_datadir = %(scrubber_datadir)s +image_cache_datadir = %(image_cache_datadir)s [pipeline:glance-api] -pipeline = versionnegotiation context apiv1app +pipeline = versionnegotiation context %(cache_pipeline)s apiv1app [pipeline:versions] pipeline = versionsapp @@ -218,6 +222,9 @@ paste.app_factory = glance.api.v1:app_factory [filter:versionnegotiation] paste.filter_factory = glance.api.middleware.version_negotiation:filter_factory +[filter:cache] +paste.filter_factory = glance.api.middleware.cache:filter_factory + [filter:context] paste.filter_factory = glance.common.context:filter_factory """ diff --git a/glance/tests/functional/test_image_cache.py b/glance/tests/functional/test_image_cache.py new file mode 100644 index 0000000000..09022a8fdf --- /dev/null +++ b/glance/tests/functional/test_image_cache.py @@ -0,0 +1,94 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Tests a Glance API server which uses the caching middleware. We +use the filesystem store, but that is really not relevant, as the +image cache is transparent to the backend store. +""" + +import hashlib +import json +import os +import unittest + +import httplib2 + +from glance.tests.functional import test_api +from glance.tests.utils import execute, skip_if_disabled + + +FIVE_KB = 5 * 1024 + + +class TestImageCache(test_api.TestApi): + + """Functional tests that exercise the image cache""" + + @skip_if_disabled + def test_cache_middleware_transparent(self): + """ + We test that putting the cache middleware into the + application pipeline gives us transparent image caching + """ + self.cleanup() + self.cache_pipeline = "cache" + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + # Verify no image 1 + path = "http://%s:%d/v1/images/1" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'HEAD') + self.assertEqual(response.status, 404) + + # Add an image and verify a 200 OK is returned + image_data = "*" * FIVE_KB + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'Image1', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers, + body=image_data) + self.assertEqual(response.status, 201) + data = json.loads(content) + self.assertEqual(data['image']['checksum'], + hashlib.md5(image_data).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "Image1") + self.assertEqual(data['image']['is_public'], True) + + # Verify image not in cache + image_cached_path = os.path.join(self.api_server.image_cache_datadir, + '1') + self.assertFalse(os.path.exists(image_cached_path)) + + # Grab the image + path = "http://%s:%d/v1/images/1" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + + # Verify image now in cache + image_cached_path = os.path.join(self.api_server.image_cache_datadir, + '1') + self.assertTrue(os.path.exists(image_cached_path)) + + self.stop_servers() diff --git a/glance/tests/stubs.py b/glance/tests/stubs.py index 3d46bdd0e4..30d084f9fb 100644 --- a/glance/tests/stubs.py +++ b/glance/tests/stubs.py @@ -197,3 +197,65 @@ def stub_out_registry_and_store_server(stubs): fake_get_connection_type) stubs.Set(glance.common.client.ImageBodyIterator, '__iter__', fake_image_iter) + + +def stub_out_registry_server(stubs): + """ + Mocks calls to 127.0.0.1 on 9191 for testing so + that a real Glance Registry server does not need to be up and + running + """ + + class FakeRegistryConnection(object): + + def __init__(self, *args, **kwargs): + pass + + def connect(self): + return True + + def close(self): + return True + + def request(self, method, url, body=None, headers={}): + self.req = webob.Request.blank("/" + url.lstrip("/")) + self.req.method = method + if headers: + self.req.headers = headers + if body: + self.req.body = body + + def getresponse(self): + sql_connection = os.environ.get('GLANCE_SQL_CONNECTION', + "sqlite:///") + context_class = 'glance.registry.context.RequestContext' + options = {'sql_connection': sql_connection, 'verbose': VERBOSE, + 'debug': DEBUG, 'context_class': context_class} + api = context.ContextMiddleware(rserver.API(options), options) + res = self.req.get_response(api) + + # httplib.Response has a read() method...fake it out + def fake_reader(): + return res.body + + setattr(res, 'read', fake_reader) + return res + + def fake_get_connection_type(client): + """ + Returns the proper connection type + """ + DEFAULT_REGISTRY_PORT = 9191 + + if (client.port == DEFAULT_REGISTRY_PORT and + client.host == '0.0.0.0'): + return FakeRegistryConnection + + def fake_image_iter(self): + for i in self.response.app_iter: + yield i + + stubs.Set(glance.common.client.BaseClient, 'get_connection_type', + fake_get_connection_type) + stubs.Set(glance.common.client.ImageBodyIterator, '__iter__', + fake_image_iter) diff --git a/glance/tests/unit/test_cache_middleware.py b/glance/tests/unit/test_cache_middleware.py new file mode 100644 index 0000000000..1992382179 --- /dev/null +++ b/glance/tests/unit/test_cache_middleware.py @@ -0,0 +1,114 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import httplib +import os +import random +import shutil +import unittest + +import stubout +import webob + +from glance import registry +from glance.api import v1 as server +from glance.api.middleware import cache +from glance.common import context +from glance.tests import stubs + +FIXTURE_DATA = '*' * 1024 + + +class TestCacheMiddleware(unittest.TestCase): + + """Test case for the cache middleware""" + + def setUp(self): + self.cache_dir = os.path.join("/", "tmp", "test.cache.%d" % + random.randint(0, 1000000)) + self.filesystem_store_datadir = os.path.join(self.cache_dir, + 'filestore') + self.options = { + 'verbose': True, + 'debug': True, + 'image_cache_datadir': self.cache_dir, + 'registry_host': '0.0.0.0', + 'registry_port': 9191, + 'default_store': 'file', + 'filesystem_store_datadir': self.filesystem_store_datadir + } + self.cache_filter = cache.CacheFilter( + server.API(self.options), self.options) + self.api = context.ContextMiddleware(self.cache_filter, self.options) + self.stubs = stubout.StubOutForTesting() + stubs.stub_out_registry_server(self.stubs) + + def tearDown(self): + self.stubs.UnsetAll() + if os.path.exists(self.cache_dir): + shutil.rmtree(self.cache_dir) + + def test_cache_image(self): + """ + Verify no images cached at start, then request an image, + and verify the image is in the cache afterwards + """ + image_cached_path = os.path.join(self.cache_dir, '1') + + self.assertFalse(os.path.exists(image_cached_path)) + + req = webob.Request.blank('/images/1') + res = req.get_response(self.api) + self.assertEquals(404, res.status_int) + + fixture_headers = {'x-image-meta-store': 'file', + 'x-image-meta-disk-format': 'vhd', + 'x-image-meta-container-format': 'ovf', + 'x-image-meta-name': 'fake image #1'} + + req = webob.Request.blank("/images") + req.method = 'POST' + for k, v in fixture_headers.iteritems(): + req.headers[k] = v + + req.headers['Content-Type'] = 'application/octet-stream' + req.body = FIXTURE_DATA + res = req.get_response(self.api) + self.assertEquals(res.status_int, httplib.CREATED) + + req = webob.Request.blank('/images/1') + res = req.get_response(self.api) + self.assertEquals(200, res.status_int) + + for chunk in res.body: + pass # We do this to trigger tee'ing the file + + self.assertTrue(os.path.exists(image_cached_path)) + self.assertEqual(0, self.cache_filter.cache.get_hit_count('1')) + + # Now verify that the next call to GET /images/1 + # yields the image from the cache... + + req = webob.Request.blank('/images/1') + res = req.get_response(self.api) + self.assertEquals(200, res.status_int) + + for chunk in res.body: + pass # We do this to trigger a hit read + + self.assertTrue(os.path.exists(image_cached_path)) + self.assertEqual(1, self.cache_filter.cache.get_hit_count('1')) diff --git a/glance/tests/unit/test_image_cache.py b/glance/tests/unit/test_image_cache.py index c28b4c77f1..efcbbd8c18 100644 --- a/glance/tests/unit/test_image_cache.py +++ b/glance/tests/unit/test_image_cache.py @@ -14,42 +14,223 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + +import os +import random +import shutil +import StringIO import unittest -import stubout - from glance import image_cache +from glance.common import exception - -def stub_out_image_cache(stubs): - def fake_make_cache_directory_if_needed(*args, **kwargs): - pass - - stubs.Set(image_cache.ImageCache, - '_make_cache_directory_if_needed', fake_make_cache_directory_if_needed) +FIXTURE_DATA = '*' * 1024 class TestImageCache(unittest.TestCase): def setUp(self): - self.stubs = stubout.StubOutForTesting() - stub_out_image_cache(self.stubs) + self.cache_dir = os.path.join("/", "tmp", "test.cache.%d" % + random.randint(0, 1000000)) + self.options = {'image_cache_datadir': self.cache_dir} + self.cache = image_cache.ImageCache(self.options) def tearDown(self): - self.stubs.UnsetAll() + if os.path.exists(self.cache_dir): + shutil.rmtree(self.cache_dir) - def test_enabled_defaults_to_false(self): - options = {} - cache = image_cache.ImageCache(options) - self.assertEqual(cache.enabled, False) + def test_auto_properties(self): + """ + Test that the auto-assigned properties are correct + """ + self.assertEqual(self.cache.path, self.cache_dir) + self.assertEqual(self.cache.invalid_path, + os.path.join(self.cache_dir, + 'invalid')) + self.assertEqual(self.cache.incomplete_path, + os.path.join(self.cache_dir, + 'incomplete')) + self.assertEqual(self.cache.prefetch_path, + os.path.join(self.cache_dir, + 'prefetch')) + self.assertEqual(self.cache.prefetching_path, + os.path.join(self.cache_dir, + 'prefetching')) - def test_can_be_disabled(self): - options = {'image_cache_enabled': 'False', - 'image_cache_datadir': '/some/place'} - cache = image_cache.ImageCache(options) - self.assertEqual(cache.enabled, False) + def test_hit(self): + """ + Verify hit(1) returns 0, then add something to the cache + and verify hit(1) returns 1. + """ + meta = {'id': 1, + 'name': 'Image1', + 'size': len(FIXTURE_DATA)} - def test_can_be_enabled(self): - options = {'image_cache_enabled': 'True', - 'image_cache_datadir': '/some/place'} - cache = image_cache.ImageCache(options) - self.assertEqual(cache.enabled, True) + self.assertFalse(self.cache.hit(1)) + + with self.cache.open(meta, 'wb') as cache_file: + cache_file.write(FIXTURE_DATA) + + self.assertTrue(self.cache.hit(1)) + + def test_bad_open_mode(self): + """ + Test than an exception is raised if attempting to open + the cache file context manager with an invalid mode string + """ + meta = {'id': 1, + 'name': 'Image1', + 'size': len(FIXTURE_DATA)} + + bad_modes = ('xb', 'wa', 'rw') + for mode in bad_modes: + exc_raised = False + try: + with self.cache.open(meta, 'xb') as cache_file: + cache_file.write(FIXTURE_DATA) + except: + exc_raised = True + self.assertTrue(exc_raised, + 'Using mode %s, failed to raise exception.' % mode) + + def test_read(self): + """ + Verify hit(1) returns 0, then add something to the cache + and verify after a subsequent read from the cache that + hit(1) returns 1. + """ + meta = {'id': 1, + 'name': 'Image1', + 'size': len(FIXTURE_DATA)} + + self.assertFalse(self.cache.hit(1)) + + with self.cache.open(meta, 'wb') as cache_file: + cache_file.write(FIXTURE_DATA) + + buff = StringIO.StringIO() + with self.cache.open(meta, 'rb') as cache_file: + for chunk in cache_file: + buff.write(chunk) + + self.assertEqual(FIXTURE_DATA, buff.getvalue()) + + def test_open_for_read(self): + """ + Test convenience wrapper for opening a cache file via + its image identifier. + """ + meta = {'id': 1, + 'name': 'Image1', + 'size': len(FIXTURE_DATA)} + + self.assertFalse(self.cache.hit(1)) + + with self.cache.open(meta, 'wb') as cache_file: + cache_file.write(FIXTURE_DATA) + + buff = StringIO.StringIO() + with self.cache.open_for_read(1) as cache_file: + for chunk in cache_file: + buff.write(chunk) + + self.assertEqual(FIXTURE_DATA, buff.getvalue()) + + def test_purge(self): + """ + Test purge method that removes an image from the cache + """ + meta = {'id': 1, + 'name': 'Image1', + 'size': len(FIXTURE_DATA)} + + self.assertFalse(self.cache.hit(1)) + + with self.cache.open(meta, 'wb') as cache_file: + cache_file.write(FIXTURE_DATA) + + self.assertTrue(self.cache.hit(1)) + + self.cache.purge(1) + + self.assertFalse(self.cache.hit(1)) + + def test_clear(self): + """ + Test purge method that removes an image from the cache + """ + metas = [ + {'id': 1, + 'name': 'Image1', + 'size': len(FIXTURE_DATA)}, + {'id': 2, + 'name': 'Image2', + 'size': len(FIXTURE_DATA)}] + + for image_id in (1, 2): + self.assertFalse(self.cache.hit(image_id)) + + for meta in metas: + with self.cache.open(meta, 'wb') as cache_file: + cache_file.write(FIXTURE_DATA) + + for image_id in (1, 2): + self.assertTrue(self.cache.hit(image_id)) + + self.cache.clear() + + for image_id in (1, 2): + self.assertFalse(self.cache.hit(image_id)) + + def test_prefetch(self): + """ + Test that queueing for prefetch and prefetching works properly + """ + meta = {'id': 1, + 'name': 'Image1', + 'size': len(FIXTURE_DATA)} + + self.assertFalse(self.cache.hit(1)) + + self.cache.queue_prefetch(meta) + + self.assertFalse(self.cache.hit(1)) + + # Test that an exception is raised if we try to queue the + # same image for prefetching + self.assertRaises(exception.Invalid, self.cache.queue_prefetch, + meta) + + self.cache.delete_queued_prefetch_image(1) + + self.assertFalse(self.cache.hit(1)) + + # Test that an exception is raised if we try to queue for + # prefetching an image that has already been cached + + with self.cache.open(meta, 'wb') as cache_file: + cache_file.write(FIXTURE_DATA) + + self.assertTrue(self.cache.hit(1)) + + self.assertRaises(exception.Invalid, self.cache.queue_prefetch, + meta) + + self.cache.purge(1) + + # We can't prefetch an image that has not been queued + # for prefetching + self.assertRaises(OSError, self.cache.do_prefetch, 1) + + self.cache.queue_prefetch(meta) + + self.assertTrue(self.cache.is_image_queued_for_prefetch(1)) + + self.assertFalse(self.cache.is_currently_prefetching_any_images()) + self.assertFalse(self.cache.is_image_currently_prefetching(1)) + + self.assertEqual(str(1), self.cache.pop_prefetch_item()) + + self.cache.do_prefetch(1) + self.assertFalse(self.cache.is_image_queued_for_prefetch(1)) + self.assertTrue(self.cache.is_currently_prefetching_any_images()) + self.assertTrue(self.cache.is_image_currently_prefetching(1)) diff --git a/glance/utils.py b/glance/utils.py index d4e2be2d77..264de5b321 100644 --- a/glance/utils.py +++ b/glance/utils.py @@ -107,20 +107,6 @@ def has_body(req): return req.content_length or 'transfer-encoding' in req.headers -def chunkiter(fp, chunk_size=65536): - """Return an iterator to a file-like obj which yields fixed size chunks - - :param fp: a file-like object - :param chunk_size: maximum size of chunk - """ - while True: - chunk = fp.read(chunk_size) - if chunk: - yield chunk - else: - break - - class PrettyTable(object): """Creates an ASCII art table for use in bin/glance