OpenStack Image Management (Glance)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

cache.py 14KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343
  1. # Copyright 2011 OpenStack Foundation
  2. # All Rights Reserved.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. """
  16. Transparent image file caching middleware, designed to live on
  17. Glance API nodes. When images are requested from the API node,
  18. this middleware caches the returned image file to local filesystem.
  19. When subsequent requests for the same image file are received,
  20. the local cached copy of the image file is returned.
  21. """
  22. import re
  23. import six
  24. from oslo_log import log as logging
  25. from six.moves import http_client as http
  26. import webob
  27. from glance.api.common import size_checked_iter
  28. from glance.api import policy
  29. from glance.api.v1 import images
  30. from glance.common import exception
  31. from glance.common import utils
  32. from glance.common import wsgi
  33. import glance.db
  34. from glance.i18n import _LE, _LI
  35. from glance import image_cache
  36. from glance import notifier
  37. import glance.registry.client.v1.api as registry
  38. LOG = logging.getLogger(__name__)
  39. PATTERNS = {
  40. ('v1', 'GET'): re.compile(r'^/v1/images/([^\/]+)$'),
  41. ('v1', 'DELETE'): re.compile(r'^/v1/images/([^\/]+)$'),
  42. ('v2', 'GET'): re.compile(r'^/v2/images/([^\/]+)/file$'),
  43. ('v2', 'DELETE'): re.compile(r'^/v2/images/([^\/]+)$')
  44. }
  45. class CacheFilter(wsgi.Middleware):
  46. def __init__(self, app):
  47. self.cache = image_cache.ImageCache()
  48. self.serializer = images.ImageSerializer()
  49. self.policy = policy.Enforcer()
  50. LOG.info(_LI("Initialized image cache middleware"))
  51. super(CacheFilter, self).__init__(app)
  52. def _verify_metadata(self, image_meta):
  53. """
  54. Sanity check the 'deleted' and 'size' metadata values.
  55. """
  56. # NOTE: admins can see image metadata in the v1 API, but shouldn't
  57. # be able to download the actual image data.
  58. if image_meta['status'] == 'deleted' and image_meta['deleted']:
  59. raise exception.NotFound()
  60. if not image_meta['size']:
  61. # override image size metadata with the actual cached
  62. # file size, see LP Bug #900959
  63. if not isinstance(image_meta, policy.ImageTarget):
  64. image_meta['size'] = self.cache.get_image_size(
  65. image_meta['id'])
  66. else:
  67. image_meta.target.size = self.cache.get_image_size(
  68. image_meta['id'])
  69. @staticmethod
  70. def _match_request(request):
  71. """Determine the version of the url and extract the image id
  72. :returns: tuple of version and image id if the url is a cacheable,
  73. otherwise None
  74. """
  75. for ((version, method), pattern) in PATTERNS.items():
  76. if request.method != method:
  77. continue
  78. match = pattern.match(request.path_info)
  79. if match is None:
  80. continue
  81. image_id = match.group(1)
  82. # Ensure the image id we got looks like an image id to filter
  83. # out a URI like /images/detail. See LP Bug #879136
  84. if image_id != 'detail':
  85. return (version, method, image_id)
  86. def _enforce(self, req, action, target=None):
  87. """Authorize an action against our policies"""
  88. if target is None:
  89. target = {}
  90. try:
  91. self.policy.enforce(req.context, action, target)
  92. except exception.Forbidden as e:
  93. LOG.debug("User not permitted to perform '%s' action", action)
  94. raise webob.exc.HTTPForbidden(explanation=e.msg, request=req)
  95. def _get_v1_image_metadata(self, request, image_id):
  96. """
  97. Retrieves image metadata using registry for v1 api and creates
  98. dictionary-like mash-up of image core and custom properties.
  99. """
  100. try:
  101. image_metadata = registry.get_image_metadata(request.context,
  102. image_id)
  103. return utils.create_mashup_dict(image_metadata)
  104. except exception.NotFound as e:
  105. LOG.debug("No metadata found for image '%s'", image_id)
  106. raise webob.exc.HTTPNotFound(explanation=e.msg, request=request)
  107. def _get_v2_image_metadata(self, request, image_id):
  108. """
  109. Retrieves image and for v2 api and creates adapter like object
  110. to access image core or custom properties on request.
  111. """
  112. db_api = glance.db.get_api()
  113. image_repo = glance.db.ImageRepo(request.context, db_api)
  114. try:
  115. image = image_repo.get(image_id)
  116. # Storing image object in request as it is required in
  117. # _process_v2_request call.
  118. request.environ['api.cache.image'] = image
  119. return policy.ImageTarget(image)
  120. except exception.NotFound as e:
  121. raise webob.exc.HTTPNotFound(explanation=e.msg, request=request)
  122. def process_request(self, request):
  123. """
  124. For requests for an image file, we check the local image
  125. cache. If present, we return the image file, appending
  126. the image metadata in headers. If not present, we pass
  127. the request on to the next application in the pipeline.
  128. """
  129. match = self._match_request(request)
  130. try:
  131. (version, method, image_id) = match
  132. except TypeError:
  133. # Trying to unpack None raises this exception
  134. return None
  135. self._stash_request_info(request, image_id, method, version)
  136. # Partial image download requests shall not be served from cache
  137. # Bug: 1664709
  138. # TODO(dharinic): If an image is already cached, add support to serve
  139. # only the requested bytes (partial image download) from the cache.
  140. if (request.headers.get('Content-Range') or
  141. request.headers.get('Range')):
  142. return None
  143. if request.method != 'GET' or not self.cache.is_cached(image_id):
  144. return None
  145. method = getattr(self, '_get_%s_image_metadata' % version)
  146. image_metadata = method(request, image_id)
  147. # Deactivated images shall not be served from cache
  148. if image_metadata['status'] == 'deactivated':
  149. return None
  150. try:
  151. self._enforce(request, 'download_image', target=image_metadata)
  152. except exception.Forbidden:
  153. return None
  154. LOG.debug("Cache hit for image '%s'", image_id)
  155. image_iterator = self.get_from_cache(image_id)
  156. method = getattr(self, '_process_%s_request' % version)
  157. try:
  158. return method(request, image_id, image_iterator, image_metadata)
  159. except exception.ImageNotFound:
  160. msg = _LE("Image cache contained image file for image '%s', "
  161. "however the registry did not contain metadata for "
  162. "that image!") % image_id
  163. LOG.error(msg)
  164. self.cache.delete_cached_image(image_id)
  165. @staticmethod
  166. def _stash_request_info(request, image_id, method, version):
  167. """
  168. Preserve the image id, version and request method for later retrieval
  169. """
  170. request.environ['api.cache.image_id'] = image_id
  171. request.environ['api.cache.method'] = method
  172. request.environ['api.cache.version'] = version
  173. @staticmethod
  174. def _fetch_request_info(request):
  175. """
  176. Preserve the cached image id, version for consumption by the
  177. process_response method of this middleware
  178. """
  179. try:
  180. image_id = request.environ['api.cache.image_id']
  181. method = request.environ['api.cache.method']
  182. version = request.environ['api.cache.version']
  183. except KeyError:
  184. return None
  185. else:
  186. return (image_id, method, version)
  187. def _process_v1_request(self, request, image_id, image_iterator,
  188. image_meta):
  189. # Don't display location
  190. if 'location' in image_meta:
  191. del image_meta['location']
  192. image_meta.pop('location_data', None)
  193. self._verify_metadata(image_meta)
  194. response = webob.Response(request=request)
  195. raw_response = {
  196. 'image_iterator': image_iterator,
  197. 'image_meta': image_meta,
  198. }
  199. return self.serializer.show(response, raw_response)
  200. def _process_v2_request(self, request, image_id, image_iterator,
  201. image_meta):
  202. # We do some contortions to get the image_metadata so
  203. # that we can provide it to 'size_checked_iter' which
  204. # will generate a notification.
  205. # TODO(mclaren): Make notification happen more
  206. # naturally once caching is part of the domain model.
  207. image = request.environ['api.cache.image']
  208. self._verify_metadata(image_meta)
  209. response = webob.Response(request=request)
  210. response.app_iter = size_checked_iter(response, image_meta,
  211. image_meta['size'],
  212. image_iterator,
  213. notifier.Notifier())
  214. # NOTE (flwang): Set the content-type, content-md5 and content-length
  215. # explicitly to be consistent with the non-cache scenario.
  216. # Besides, it's not worth the candle to invoke the "download" method
  217. # of ResponseSerializer under image_data. Because method "download"
  218. # will reset the app_iter. Then we have to call method
  219. # "size_checked_iter" to avoid missing any notification. But after
  220. # call "size_checked_iter", we will lose the content-md5 and
  221. # content-length got by the method "download" because of this issue:
  222. # https://github.com/Pylons/webob/issues/86
  223. response.headers['Content-Type'] = 'application/octet-stream'
  224. if image.checksum:
  225. response.headers['Content-MD5'] = (image.checksum.encode('utf-8')
  226. if six.PY2 else image.checksum)
  227. response.headers['Content-Length'] = str(image.size)
  228. return response
  229. def process_response(self, resp):
  230. """
  231. We intercept the response coming back from the main
  232. images Resource, removing image file from the cache
  233. if necessary
  234. """
  235. status_code = self.get_status_code(resp)
  236. if not 200 <= status_code < 300:
  237. return resp
  238. # Note(dharinic): Bug: 1664709: Do not cache partial images.
  239. if status_code == http.PARTIAL_CONTENT:
  240. return resp
  241. try:
  242. (image_id, method, version) = self._fetch_request_info(
  243. resp.request)
  244. except TypeError:
  245. return resp
  246. if method == 'GET' and status_code == http.NO_CONTENT:
  247. # Bugfix:1251055 - Don't cache non-existent image files.
  248. # NOTE: Both GET for an image without locations and DELETE return
  249. # 204 but DELETE should be processed.
  250. return resp
  251. method_str = '_process_%s_response' % method
  252. try:
  253. process_response_method = getattr(self, method_str)
  254. except AttributeError:
  255. LOG.error(_LE('could not find %s') % method_str)
  256. # Nothing to do here, move along
  257. return resp
  258. else:
  259. return process_response_method(resp, image_id, version=version)
  260. def _process_DELETE_response(self, resp, image_id, version=None):
  261. if self.cache.is_cached(image_id):
  262. LOG.debug("Removing image %s from cache", image_id)
  263. self.cache.delete_cached_image(image_id)
  264. return resp
  265. def _process_GET_response(self, resp, image_id, version=None):
  266. image_checksum = resp.headers.get('Content-MD5')
  267. if not image_checksum:
  268. # API V1 stores the checksum in a different header:
  269. image_checksum = resp.headers.get('x-image-meta-checksum')
  270. if not image_checksum:
  271. LOG.error(_LE("Checksum header is missing."))
  272. # fetch image_meta on the basis of version
  273. image_metadata = None
  274. if version:
  275. method = getattr(self, '_get_%s_image_metadata' % version)
  276. image_metadata = method(resp.request, image_id)
  277. # NOTE(zhiyan): image_cache return a generator object and set to
  278. # response.app_iter, it will be called by eventlet.wsgi later.
  279. # So we need enforce policy firstly but do it by application
  280. # since eventlet.wsgi could not catch webob.exc.HTTPForbidden and
  281. # return 403 error to client then.
  282. self._enforce(resp.request, 'download_image', target=image_metadata)
  283. resp.app_iter = self.cache.get_caching_iter(image_id, image_checksum,
  284. resp.app_iter)
  285. return resp
  286. def get_status_code(self, response):
  287. """
  288. Returns the integer status code from the response, which
  289. can be either a Webob.Response (used in testing) or httplib.Response
  290. """
  291. if hasattr(response, 'status_int'):
  292. return response.status_int
  293. return response.status
  294. def get_from_cache(self, image_id):
  295. """Called if cache hit"""
  296. with self.cache.open_for_read(image_id) as cache_file:
  297. chunks = utils.chunkiter(cache_file)
  298. for chunk in chunks:
  299. yield chunk