OpenStack Image Management (Glance)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

test_image_cache.py 20KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565
  1. # Copyright 2011 OpenStack Foundation
  2. # All Rights Reserved.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. from __future__ import absolute_import
  16. from contextlib import contextmanager
  17. import datetime
  18. import hashlib
  19. import os
  20. import time
  21. import fixtures
  22. from oslo_utils import units
  23. from oslotest import moxstubout
  24. import six
  25. # NOTE(jokke): simplified transition to py3, behaves like py2 xrange
  26. from six.moves import range
  27. from glance.common import exception
  28. from glance import image_cache
  29. # NOTE(bcwaldon): This is imported to load the registry config options
  30. import glance.registry # noqa
  31. from glance.tests import utils as test_utils
  32. from glance.tests.utils import skip_if_disabled
  33. from glance.tests.utils import xattr_writes_supported
  34. FIXTURE_LENGTH = 1024
  35. FIXTURE_DATA = b'*' * FIXTURE_LENGTH
  36. class ImageCacheTestCase(object):
  37. def _setup_fixture_file(self):
  38. FIXTURE_FILE = six.BytesIO(FIXTURE_DATA)
  39. self.assertFalse(self.cache.is_cached(1))
  40. self.assertTrue(self.cache.cache_image_file(1, FIXTURE_FILE))
  41. self.assertTrue(self.cache.is_cached(1))
  42. @skip_if_disabled
  43. def test_is_cached(self):
  44. """Verify is_cached(1) returns 0, then add something to the cache
  45. and verify is_cached(1) returns 1.
  46. """
  47. self._setup_fixture_file()
  48. @skip_if_disabled
  49. def test_read(self):
  50. """Verify is_cached(1) returns 0, then add something to the cache
  51. and verify after a subsequent read from the cache that
  52. is_cached(1) returns 1.
  53. """
  54. self._setup_fixture_file()
  55. buff = six.BytesIO()
  56. with self.cache.open_for_read(1) as cache_file:
  57. for chunk in cache_file:
  58. buff.write(chunk)
  59. self.assertEqual(FIXTURE_DATA, buff.getvalue())
  60. @skip_if_disabled
  61. def test_open_for_read(self):
  62. """Test convenience wrapper for opening a cache file via
  63. its image identifier.
  64. """
  65. self._setup_fixture_file()
  66. buff = six.BytesIO()
  67. with self.cache.open_for_read(1) as cache_file:
  68. for chunk in cache_file:
  69. buff.write(chunk)
  70. self.assertEqual(FIXTURE_DATA, buff.getvalue())
  71. @skip_if_disabled
  72. def test_get_image_size(self):
  73. """Test convenience wrapper for querying cache file size via
  74. its image identifier.
  75. """
  76. self._setup_fixture_file()
  77. size = self.cache.get_image_size(1)
  78. self.assertEqual(FIXTURE_LENGTH, size)
  79. @skip_if_disabled
  80. def test_delete(self):
  81. """Test delete method that removes an image from the cache."""
  82. self._setup_fixture_file()
  83. self.cache.delete_cached_image(1)
  84. self.assertFalse(self.cache.is_cached(1))
  85. @skip_if_disabled
  86. def test_delete_all(self):
  87. """Test delete method that removes an image from the cache."""
  88. for image_id in (1, 2):
  89. self.assertFalse(self.cache.is_cached(image_id))
  90. for image_id in (1, 2):
  91. FIXTURE_FILE = six.BytesIO(FIXTURE_DATA)
  92. self.assertTrue(self.cache.cache_image_file(image_id,
  93. FIXTURE_FILE))
  94. for image_id in (1, 2):
  95. self.assertTrue(self.cache.is_cached(image_id))
  96. self.cache.delete_all_cached_images()
  97. for image_id in (1, 2):
  98. self.assertFalse(self.cache.is_cached(image_id))
  99. @skip_if_disabled
  100. def test_clean_stalled(self):
  101. """Test the clean method removes expected images."""
  102. incomplete_file_path = os.path.join(self.cache_dir, 'incomplete', '1')
  103. incomplete_file = open(incomplete_file_path, 'wb')
  104. incomplete_file.write(FIXTURE_DATA)
  105. incomplete_file.close()
  106. self.assertTrue(os.path.exists(incomplete_file_path))
  107. self.cache.clean(stall_time=0)
  108. self.assertFalse(os.path.exists(incomplete_file_path))
  109. @skip_if_disabled
  110. def test_clean_stalled_nonzero_stall_time(self):
  111. """
  112. Test the clean method removes the stalled images as expected
  113. """
  114. incomplete_file_path_1 = os.path.join(self.cache_dir,
  115. 'incomplete', '1')
  116. incomplete_file_path_2 = os.path.join(self.cache_dir,
  117. 'incomplete', '2')
  118. for f in (incomplete_file_path_1, incomplete_file_path_2):
  119. incomplete_file = open(f, 'wb')
  120. incomplete_file.write(FIXTURE_DATA)
  121. incomplete_file.close()
  122. mtime = os.path.getmtime(incomplete_file_path_1)
  123. pastday = (datetime.datetime.fromtimestamp(mtime) -
  124. datetime.timedelta(days=1))
  125. atime = int(time.mktime(pastday.timetuple()))
  126. mtime = atime
  127. os.utime(incomplete_file_path_1, (atime, mtime))
  128. self.assertTrue(os.path.exists(incomplete_file_path_1))
  129. self.assertTrue(os.path.exists(incomplete_file_path_2))
  130. self.cache.clean(stall_time=3600)
  131. self.assertFalse(os.path.exists(incomplete_file_path_1))
  132. self.assertTrue(os.path.exists(incomplete_file_path_2))
  133. @skip_if_disabled
  134. def test_prune(self):
  135. """
  136. Test that pruning the cache works as expected...
  137. """
  138. self.assertEqual(0, self.cache.get_cache_size())
  139. # Add a bunch of images to the cache. The max cache size for the cache
  140. # is set to 5KB and each image is 1K. We use 11 images in this test.
  141. # The first 10 are added to and retrieved from cache in the same order.
  142. # Then, the 11th image is added to cache but not retrieved before we
  143. # prune. We should see only 5 images left after pruning, and the
  144. # images that are least recently accessed should be the ones pruned...
  145. for x in range(10):
  146. FIXTURE_FILE = six.BytesIO(FIXTURE_DATA)
  147. self.assertTrue(self.cache.cache_image_file(x, FIXTURE_FILE))
  148. self.assertEqual(10 * units.Ki, self.cache.get_cache_size())
  149. # OK, hit the images that are now cached...
  150. for x in range(10):
  151. buff = six.BytesIO()
  152. with self.cache.open_for_read(x) as cache_file:
  153. for chunk in cache_file:
  154. buff.write(chunk)
  155. # Add a new image to cache.
  156. # This is specifically to test the bug: 1438564
  157. FIXTURE_FILE = six.BytesIO(FIXTURE_DATA)
  158. self.assertTrue(self.cache.cache_image_file(99, FIXTURE_FILE))
  159. self.cache.prune()
  160. self.assertEqual(5 * units.Ki, self.cache.get_cache_size())
  161. # Ensure images 0, 1, 2, 3, 4 & 5 are not cached anymore
  162. for x in range(0, 6):
  163. self.assertFalse(self.cache.is_cached(x),
  164. "Image %s was cached!" % x)
  165. # Ensure images 6, 7, 8 and 9 are still cached
  166. for x in range(6, 10):
  167. self.assertTrue(self.cache.is_cached(x),
  168. "Image %s was not cached!" % x)
  169. # Ensure the newly added image, 99, is still cached
  170. self.assertTrue(self.cache.is_cached(99), "Image 99 was not cached!")
  171. @skip_if_disabled
  172. def test_prune_to_zero(self):
  173. """Test that an image_cache_max_size of 0 doesn't kill the pruner
  174. This is a test specifically for LP #1039854
  175. """
  176. self.assertEqual(0, self.cache.get_cache_size())
  177. FIXTURE_FILE = six.BytesIO(FIXTURE_DATA)
  178. self.assertTrue(self.cache.cache_image_file('xxx', FIXTURE_FILE))
  179. self.assertEqual(1024, self.cache.get_cache_size())
  180. # OK, hit the image that is now cached...
  181. buff = six.BytesIO()
  182. with self.cache.open_for_read('xxx') as cache_file:
  183. for chunk in cache_file:
  184. buff.write(chunk)
  185. self.config(image_cache_max_size=0)
  186. self.cache.prune()
  187. self.assertEqual(0, self.cache.get_cache_size())
  188. self.assertFalse(self.cache.is_cached('xxx'))
  189. @skip_if_disabled
  190. def test_queue(self):
  191. """
  192. Test that queueing works properly
  193. """
  194. self.assertFalse(self.cache.is_cached(1))
  195. self.assertFalse(self.cache.is_queued(1))
  196. FIXTURE_FILE = six.BytesIO(FIXTURE_DATA)
  197. self.assertTrue(self.cache.queue_image(1))
  198. self.assertTrue(self.cache.is_queued(1))
  199. self.assertFalse(self.cache.is_cached(1))
  200. # Should not return True if the image is already
  201. # queued for caching...
  202. self.assertFalse(self.cache.queue_image(1))
  203. self.assertFalse(self.cache.is_cached(1))
  204. # Test that we return False if we try to queue
  205. # an image that has already been cached
  206. self.assertTrue(self.cache.cache_image_file(1, FIXTURE_FILE))
  207. self.assertFalse(self.cache.is_queued(1))
  208. self.assertTrue(self.cache.is_cached(1))
  209. self.assertFalse(self.cache.queue_image(1))
  210. self.cache.delete_cached_image(1)
  211. for x in range(3):
  212. self.assertTrue(self.cache.queue_image(x))
  213. self.assertEqual(['0', '1', '2'],
  214. self.cache.get_queued_images())
  215. def test_open_for_write_good(self):
  216. """
  217. Test to see if open_for_write works in normal case
  218. """
  219. # test a good case
  220. image_id = '1'
  221. self.assertFalse(self.cache.is_cached(image_id))
  222. with self.cache.driver.open_for_write(image_id) as cache_file:
  223. cache_file.write(b'a')
  224. self.assertTrue(self.cache.is_cached(image_id),
  225. "Image %s was NOT cached!" % image_id)
  226. # make sure it has tidied up
  227. incomplete_file_path = os.path.join(self.cache_dir,
  228. 'incomplete', image_id)
  229. invalid_file_path = os.path.join(self.cache_dir, 'invalid', image_id)
  230. self.assertFalse(os.path.exists(incomplete_file_path))
  231. self.assertFalse(os.path.exists(invalid_file_path))
  232. def test_open_for_write_with_exception(self):
  233. """
  234. Test to see if open_for_write works in a failure case for each driver
  235. This case is where an exception is raised while the file is being
  236. written. The image is partially filled in cache and filling wont resume
  237. so verify the image is moved to invalid/ directory
  238. """
  239. # test a case where an exception is raised while the file is open
  240. image_id = '1'
  241. self.assertFalse(self.cache.is_cached(image_id))
  242. try:
  243. with self.cache.driver.open_for_write(image_id):
  244. raise IOError
  245. except Exception as e:
  246. self.assertIsInstance(e, IOError)
  247. self.assertFalse(self.cache.is_cached(image_id),
  248. "Image %s was cached!" % image_id)
  249. # make sure it has tidied up
  250. incomplete_file_path = os.path.join(self.cache_dir,
  251. 'incomplete', image_id)
  252. invalid_file_path = os.path.join(self.cache_dir, 'invalid', image_id)
  253. self.assertFalse(os.path.exists(incomplete_file_path))
  254. self.assertTrue(os.path.exists(invalid_file_path))
  255. def test_caching_iterator(self):
  256. """
  257. Test to see if the caching iterator interacts properly with the driver
  258. When the iterator completes going through the data the driver should
  259. have closed the image and placed it correctly
  260. """
  261. # test a case where an exception NOT raised while the file is open,
  262. # and a consuming iterator completes
  263. def consume(image_id):
  264. data = [b'a', b'b', b'c', b'd', b'e', b'f']
  265. checksum = None
  266. caching_iter = self.cache.get_caching_iter(image_id, checksum,
  267. iter(data))
  268. self.assertEqual(data, list(caching_iter))
  269. image_id = '1'
  270. self.assertFalse(self.cache.is_cached(image_id))
  271. consume(image_id)
  272. self.assertTrue(self.cache.is_cached(image_id),
  273. "Image %s was NOT cached!" % image_id)
  274. # make sure it has tidied up
  275. incomplete_file_path = os.path.join(self.cache_dir,
  276. 'incomplete', image_id)
  277. invalid_file_path = os.path.join(self.cache_dir, 'invalid', image_id)
  278. self.assertFalse(os.path.exists(incomplete_file_path))
  279. self.assertFalse(os.path.exists(invalid_file_path))
  280. def test_caching_iterator_handles_backend_failure(self):
  281. """
  282. Test that when the backend fails, caching_iter does not continue trying
  283. to consume data, and rolls back the cache.
  284. """
  285. def faulty_backend():
  286. data = [b'a', b'b', b'c', b'Fail', b'd', b'e', b'f']
  287. for d in data:
  288. if d == b'Fail':
  289. raise exception.GlanceException('Backend failure')
  290. yield d
  291. def consume(image_id):
  292. caching_iter = self.cache.get_caching_iter(image_id, None,
  293. faulty_backend())
  294. # exercise the caching_iter
  295. list(caching_iter)
  296. image_id = '1'
  297. self.assertRaises(exception.GlanceException, consume, image_id)
  298. # make sure bad image was not cached
  299. self.assertFalse(self.cache.is_cached(image_id))
  300. def test_caching_iterator_falloffend(self):
  301. """
  302. Test to see if the caching iterator interacts properly with the driver
  303. in a case where the iterator is only partially consumed. In this case
  304. the image is only partially filled in cache and filling wont resume.
  305. When the iterator goes out of scope the driver should have closed the
  306. image and moved it from incomplete/ to invalid/
  307. """
  308. # test a case where a consuming iterator just stops.
  309. def falloffend(image_id):
  310. data = [b'a', b'b', b'c', b'd', b'e', b'f']
  311. checksum = None
  312. caching_iter = self.cache.get_caching_iter(image_id, checksum,
  313. iter(data))
  314. self.assertEqual(b'a', next(caching_iter))
  315. image_id = '1'
  316. self.assertFalse(self.cache.is_cached(image_id))
  317. falloffend(image_id)
  318. self.assertFalse(self.cache.is_cached(image_id),
  319. "Image %s was cached!" % image_id)
  320. # make sure it has tidied up
  321. incomplete_file_path = os.path.join(self.cache_dir,
  322. 'incomplete', image_id)
  323. invalid_file_path = os.path.join(self.cache_dir, 'invalid', image_id)
  324. self.assertFalse(os.path.exists(incomplete_file_path))
  325. self.assertTrue(os.path.exists(invalid_file_path))
  326. def test_gate_caching_iter_good_checksum(self):
  327. image = b"12345678990abcdefghijklmnop"
  328. image_id = 123
  329. md5 = hashlib.md5()
  330. md5.update(image)
  331. checksum = md5.hexdigest()
  332. cache = image_cache.ImageCache()
  333. img_iter = cache.get_caching_iter(image_id, checksum, [image])
  334. for chunk in img_iter:
  335. pass
  336. # checksum is valid, fake image should be cached:
  337. self.assertTrue(cache.is_cached(image_id))
  338. def test_gate_caching_iter_bad_checksum(self):
  339. image = b"12345678990abcdefghijklmnop"
  340. image_id = 123
  341. checksum = "foobar" # bad.
  342. cache = image_cache.ImageCache()
  343. img_iter = cache.get_caching_iter(image_id, checksum, [image])
  344. def reader():
  345. for chunk in img_iter:
  346. pass
  347. self.assertRaises(exception.GlanceException, reader)
  348. # checksum is invalid, caching will fail:
  349. self.assertFalse(cache.is_cached(image_id))
  350. class TestImageCacheXattr(test_utils.BaseTestCase,
  351. ImageCacheTestCase):
  352. """Tests image caching when xattr is used in cache"""
  353. def setUp(self):
  354. """
  355. Test to see if the pre-requisites for the image cache
  356. are working (python-xattr installed and xattr support on the
  357. filesystem)
  358. """
  359. super(TestImageCacheXattr, self).setUp()
  360. if getattr(self, 'disable', False):
  361. return
  362. self.cache_dir = self.useFixture(fixtures.TempDir()).path
  363. if not getattr(self, 'inited', False):
  364. try:
  365. import xattr # noqa
  366. except ImportError:
  367. self.inited = True
  368. self.disabled = True
  369. self.disabled_message = ("python-xattr not installed.")
  370. return
  371. self.inited = True
  372. self.disabled = False
  373. self.config(image_cache_dir=self.cache_dir,
  374. image_cache_driver='xattr',
  375. image_cache_max_size=5 * units.Ki)
  376. self.cache = image_cache.ImageCache()
  377. if not xattr_writes_supported(self.cache_dir):
  378. self.inited = True
  379. self.disabled = True
  380. self.disabled_message = ("filesystem does not support xattr")
  381. return
  382. class TestImageCacheSqlite(test_utils.BaseTestCase,
  383. ImageCacheTestCase):
  384. """Tests image caching when SQLite is used in cache"""
  385. def setUp(self):
  386. """
  387. Test to see if the pre-requisites for the image cache
  388. are working (python-sqlite3 installed)
  389. """
  390. super(TestImageCacheSqlite, self).setUp()
  391. if getattr(self, 'disable', False):
  392. return
  393. if not getattr(self, 'inited', False):
  394. try:
  395. import sqlite3 # noqa
  396. except ImportError:
  397. self.inited = True
  398. self.disabled = True
  399. self.disabled_message = ("python-sqlite3 not installed.")
  400. return
  401. self.inited = True
  402. self.disabled = False
  403. self.cache_dir = self.useFixture(fixtures.TempDir()).path
  404. self.config(image_cache_dir=self.cache_dir,
  405. image_cache_driver='sqlite',
  406. image_cache_max_size=5 * units.Ki)
  407. self.cache = image_cache.ImageCache()
  408. class TestImageCacheNoDep(test_utils.BaseTestCase):
  409. def setUp(self):
  410. super(TestImageCacheNoDep, self).setUp()
  411. self.driver = None
  412. def init_driver(self2):
  413. self2.driver = self.driver
  414. mox_fixture = self.useFixture(moxstubout.MoxStubout())
  415. self.stubs = mox_fixture.stubs
  416. self.stubs.Set(image_cache.ImageCache, 'init_driver', init_driver)
  417. def test_get_caching_iter_when_write_fails(self):
  418. class FailingFile(object):
  419. def write(self, data):
  420. if data == "Fail":
  421. raise IOError
  422. class FailingFileDriver(object):
  423. def is_cacheable(self, *args, **kwargs):
  424. return True
  425. @contextmanager
  426. def open_for_write(self, *args, **kwargs):
  427. yield FailingFile()
  428. self.driver = FailingFileDriver()
  429. cache = image_cache.ImageCache()
  430. data = [b'a', b'b', b'c', b'Fail', b'd', b'e', b'f']
  431. caching_iter = cache.get_caching_iter('dummy_id', None, iter(data))
  432. self.assertEqual(data, list(caching_iter))
  433. def test_get_caching_iter_when_open_fails(self):
  434. class OpenFailingDriver(object):
  435. def is_cacheable(self, *args, **kwargs):
  436. return True
  437. @contextmanager
  438. def open_for_write(self, *args, **kwargs):
  439. raise IOError
  440. self.driver = OpenFailingDriver()
  441. cache = image_cache.ImageCache()
  442. data = [b'a', b'b', b'c', b'd', b'e', b'f']
  443. caching_iter = cache.get_caching_iter('dummy_id', None, iter(data))
  444. self.assertEqual(data, list(caching_iter))