OpenStack Image Management (Glance)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

test_cache_middleware.py 44KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163
  1. # Copyright 2011 OpenStack Foundation
  2. # All Rights Reserved.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. """
  16. Tests a Glance API server which uses the caching middleware that
  17. uses the default SQLite cache driver. We use the filesystem store,
  18. but that is really not relevant, as the image cache is transparent
  19. to the backend store.
  20. """
  21. import hashlib
  22. import os
  23. import shutil
  24. import sys
  25. import time
  26. import uuid
  27. import httplib2
  28. from oslo_serialization import jsonutils
  29. from oslo_utils import units
  30. from six.moves import http_client
  31. # NOTE(jokke): simplified transition to py3, behaves like py2 xrange
  32. from six.moves import range
  33. from glance.tests import functional
  34. from glance.tests.functional.store_utils import get_http_uri
  35. from glance.tests.functional.store_utils import setup_http
  36. from glance.tests.utils import execute
  37. from glance.tests.utils import minimal_headers
  38. from glance.tests.utils import skip_if_disabled
  39. from glance.tests.utils import xattr_writes_supported
  40. FIVE_KB = 5 * units.Ki
  41. class BaseCacheMiddlewareTest(object):
  42. @skip_if_disabled
  43. def test_cache_middleware_transparent_v1(self):
  44. """
  45. We test that putting the cache middleware into the
  46. application pipeline gives us transparent image caching
  47. """
  48. self.cleanup()
  49. self.start_servers(**self.__dict__.copy())
  50. # Add an image and verify a 200 OK is returned
  51. image_data = b"*" * FIVE_KB
  52. headers = minimal_headers('Image1')
  53. path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
  54. http = httplib2.Http()
  55. response, content = http.request(path, 'POST', headers=headers,
  56. body=image_data)
  57. self.assertEqual(http_client.CREATED, response.status)
  58. data = jsonutils.loads(content)
  59. self.assertEqual(hashlib.md5(image_data).hexdigest(),
  60. data['image']['checksum'])
  61. self.assertEqual(FIVE_KB, data['image']['size'])
  62. self.assertEqual("Image1", data['image']['name'])
  63. self.assertTrue(data['image']['is_public'])
  64. image_id = data['image']['id']
  65. # Verify image not in cache
  66. image_cached_path = os.path.join(self.api_server.image_cache_dir,
  67. image_id)
  68. self.assertFalse(os.path.exists(image_cached_path))
  69. # Grab the image
  70. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  71. image_id)
  72. http = httplib2.Http()
  73. response, content = http.request(path, 'GET')
  74. self.assertEqual(http_client.OK, response.status)
  75. # Verify image now in cache
  76. image_cached_path = os.path.join(self.api_server.image_cache_dir,
  77. image_id)
  78. # You might wonder why the heck this is here... well, it's here
  79. # because it took me forever to figure out that the disk write
  80. # cache in Linux was causing random failures of the os.path.exists
  81. # assert directly below this. Basically, since the cache is writing
  82. # the image file to disk in a different process, the write buffers
  83. # don't flush the cache file during an os.rename() properly, resulting
  84. # in a false negative on the file existence check below. This little
  85. # loop pauses the execution of this process for no more than 1.5
  86. # seconds. If after that time the cached image file still doesn't
  87. # appear on disk, something really is wrong, and the assert should
  88. # trigger...
  89. i = 0
  90. while not os.path.exists(image_cached_path) and i < 30:
  91. time.sleep(0.05)
  92. i = i + 1
  93. self.assertTrue(os.path.exists(image_cached_path))
  94. # Now, we delete the image from the server and verify that
  95. # the image cache no longer contains the deleted image
  96. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  97. image_id)
  98. http = httplib2.Http()
  99. response, content = http.request(path, 'DELETE')
  100. self.assertEqual(http_client.OK, response.status)
  101. self.assertFalse(os.path.exists(image_cached_path))
  102. self.stop_servers()
  103. @skip_if_disabled
  104. def test_cache_middleware_transparent_v2(self):
  105. """Ensure the v2 API image transfer calls trigger caching"""
  106. self.cleanup()
  107. self.start_servers(**self.__dict__.copy())
  108. # Add an image and verify success
  109. path = "http://%s:%d/v2/images" % ("0.0.0.0", self.api_port)
  110. http = httplib2.Http()
  111. headers = {'content-type': 'application/json'}
  112. image_entity = {
  113. 'name': 'Image1',
  114. 'visibility': 'public',
  115. 'container_format': 'bare',
  116. 'disk_format': 'raw',
  117. }
  118. response, content = http.request(path, 'POST',
  119. headers=headers,
  120. body=jsonutils.dumps(image_entity))
  121. self.assertEqual(http_client.CREATED, response.status)
  122. data = jsonutils.loads(content)
  123. image_id = data['id']
  124. path = "http://%s:%d/v2/images/%s/file" % ("0.0.0.0", self.api_port,
  125. image_id)
  126. headers = {'content-type': 'application/octet-stream'}
  127. image_data = "*" * FIVE_KB
  128. response, content = http.request(path, 'PUT',
  129. headers=headers,
  130. body=image_data)
  131. self.assertEqual(http_client.NO_CONTENT, response.status)
  132. # Verify image not in cache
  133. image_cached_path = os.path.join(self.api_server.image_cache_dir,
  134. image_id)
  135. self.assertFalse(os.path.exists(image_cached_path))
  136. # Grab the image
  137. http = httplib2.Http()
  138. response, content = http.request(path, 'GET')
  139. self.assertEqual(http_client.OK, response.status)
  140. # Verify image now in cache
  141. image_cached_path = os.path.join(self.api_server.image_cache_dir,
  142. image_id)
  143. self.assertTrue(os.path.exists(image_cached_path))
  144. # Now, we delete the image from the server and verify that
  145. # the image cache no longer contains the deleted image
  146. path = "http://%s:%d/v2/images/%s" % ("0.0.0.0", self.api_port,
  147. image_id)
  148. http = httplib2.Http()
  149. response, content = http.request(path, 'DELETE')
  150. self.assertEqual(http_client.NO_CONTENT, response.status)
  151. self.assertFalse(os.path.exists(image_cached_path))
  152. self.stop_servers()
  153. @skip_if_disabled
  154. def test_partially_downloaded_images_are_not_cached_v2_api(self):
  155. """
  156. Verify that we do not cache images that were downloaded partially
  157. using v2 images API.
  158. """
  159. self.cleanup()
  160. self.start_servers(**self.__dict__.copy())
  161. # Add an image and verify success
  162. path = "http://%s:%d/v2/images" % ("0.0.0.0", self.api_port)
  163. http = httplib2.Http()
  164. headers = {'content-type': 'application/json'}
  165. image_entity = {
  166. 'name': 'Image1',
  167. 'visibility': 'public',
  168. 'container_format': 'bare',
  169. 'disk_format': 'raw',
  170. }
  171. response, content = http.request(path, 'POST',
  172. headers=headers,
  173. body=jsonutils.dumps(image_entity))
  174. self.assertEqual(http_client.CREATED, response.status)
  175. data = jsonutils.loads(content)
  176. image_id = data['id']
  177. path = "http://%s:%d/v2/images/%s/file" % ("0.0.0.0", self.api_port,
  178. image_id)
  179. headers = {'content-type': 'application/octet-stream'}
  180. image_data = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
  181. response, content = http.request(path, 'PUT',
  182. headers=headers,
  183. body=image_data)
  184. self.assertEqual(http_client.NO_CONTENT, response.status)
  185. # Verify that this image is not in cache
  186. image_cached_path = os.path.join(self.api_server.image_cache_dir,
  187. image_id)
  188. self.assertFalse(os.path.exists(image_cached_path))
  189. # partially download this image and verify status 206
  190. http = httplib2.Http()
  191. # range download request
  192. range_ = 'bytes=3-5'
  193. headers = {
  194. 'X-Identity-Status': 'Confirmed',
  195. 'X-Auth-Token': '932c5c84-02ac-4fe5-a9ba-620af0e2bb96',
  196. 'X-User-Id': 'f9a41d13-0c13-47e9-bee2-ce4e8bfe958e',
  197. 'X-Tenant-Id': str(uuid.uuid4()),
  198. 'X-Roles': 'member',
  199. 'Range': range_
  200. }
  201. response, content = http.request(path, 'GET', headers=headers)
  202. self.assertEqual(http_client.PARTIAL_CONTENT, response.status)
  203. self.assertEqual(b'DEF', content)
  204. # content-range download request
  205. # NOTE(dharinic): Glance incorrectly supports Content-Range for partial
  206. # image downloads in requests. This test is included to ensure that
  207. # we prevent regression.
  208. content_range = 'bytes 3-5/*'
  209. headers = {
  210. 'X-Identity-Status': 'Confirmed',
  211. 'X-Auth-Token': '932c5c84-02ac-4fe5-a9ba-620af0e2bb96',
  212. 'X-User-Id': 'f9a41d13-0c13-47e9-bee2-ce4e8bfe958e',
  213. 'X-Tenant-Id': str(uuid.uuid4()),
  214. 'X-Roles': 'member',
  215. 'Content-Range': content_range
  216. }
  217. response, content = http.request(path, 'GET', headers=headers)
  218. self.assertEqual(http_client.PARTIAL_CONTENT, response.status)
  219. self.assertEqual(b'DEF', content)
  220. # verify that we do not cache the partial image
  221. image_cached_path = os.path.join(self.api_server.image_cache_dir,
  222. image_id)
  223. self.assertFalse(os.path.exists(image_cached_path))
  224. self.stop_servers()
  225. @skip_if_disabled
  226. def test_partial_download_of_cached_images_v2_api(self):
  227. """
  228. Verify that partial download requests for a fully cached image
  229. succeeds; we do not serve it from cache.
  230. """
  231. self.cleanup()
  232. self.start_servers(**self.__dict__.copy())
  233. # Add an image and verify success
  234. path = "http://%s:%d/v2/images" % ("0.0.0.0", self.api_port)
  235. http = httplib2.Http()
  236. headers = {'content-type': 'application/json'}
  237. image_entity = {
  238. 'name': 'Image1',
  239. 'visibility': 'public',
  240. 'container_format': 'bare',
  241. 'disk_format': 'raw',
  242. }
  243. response, content = http.request(path, 'POST',
  244. headers=headers,
  245. body=jsonutils.dumps(image_entity))
  246. self.assertEqual(http_client.CREATED, response.status)
  247. data = jsonutils.loads(content)
  248. image_id = data['id']
  249. path = "http://%s:%d/v2/images/%s/file" % ("0.0.0.0", self.api_port,
  250. image_id)
  251. headers = {'content-type': 'application/octet-stream'}
  252. image_data = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
  253. response, content = http.request(path, 'PUT',
  254. headers=headers,
  255. body=image_data)
  256. self.assertEqual(http_client.NO_CONTENT, response.status)
  257. # Verify that this image is not in cache
  258. image_cached_path = os.path.join(self.api_server.image_cache_dir,
  259. image_id)
  260. self.assertFalse(os.path.exists(image_cached_path))
  261. # Download the entire image
  262. http = httplib2.Http()
  263. response, content = http.request(path, 'GET')
  264. self.assertEqual(http_client.OK, response.status)
  265. self.assertEqual(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ', content)
  266. # Verify that the image is now in cache
  267. image_cached_path = os.path.join(self.api_server.image_cache_dir,
  268. image_id)
  269. self.assertTrue(os.path.exists(image_cached_path))
  270. # Modify the data in cache so we can verify the partially downloaded
  271. # content was not from cache indeed.
  272. with open(image_cached_path, 'w') as cache_file:
  273. cache_file.write('0123456789')
  274. # Partially attempt a download of this image and verify that is not
  275. # from cache
  276. # range download request
  277. range_ = 'bytes=3-5'
  278. headers = {
  279. 'X-Identity-Status': 'Confirmed',
  280. 'X-Auth-Token': '932c5c84-02ac-4fe5-a9ba-620af0e2bb96',
  281. 'X-User-Id': 'f9a41d13-0c13-47e9-bee2-ce4e8bfe958e',
  282. 'X-Tenant-Id': str(uuid.uuid4()),
  283. 'X-Roles': 'member',
  284. 'Range': range_,
  285. 'content-type': 'application/json'
  286. }
  287. response, content = http.request(path, 'GET', headers=headers)
  288. self.assertEqual(http_client.PARTIAL_CONTENT, response.status)
  289. self.assertEqual(b'DEF', content)
  290. self.assertNotEqual(b'345', content)
  291. self.assertNotEqual(image_data, content)
  292. # content-range download request
  293. # NOTE(dharinic): Glance incorrectly supports Content-Range for partial
  294. # image downloads in requests. This test is included to ensure that
  295. # we prevent regression.
  296. content_range = 'bytes 3-5/*'
  297. headers = {
  298. 'X-Identity-Status': 'Confirmed',
  299. 'X-Auth-Token': '932c5c84-02ac-4fe5-a9ba-620af0e2bb96',
  300. 'X-User-Id': 'f9a41d13-0c13-47e9-bee2-ce4e8bfe958e',
  301. 'X-Tenant-Id': str(uuid.uuid4()),
  302. 'X-Roles': 'member',
  303. 'Content-Range': content_range,
  304. 'content-type': 'application/json'
  305. }
  306. response, content = http.request(path, 'GET', headers=headers)
  307. self.assertEqual(http_client.PARTIAL_CONTENT, response.status)
  308. self.assertEqual(b'DEF', content)
  309. self.assertNotEqual(b'345', content)
  310. self.assertNotEqual(image_data, content)
  311. self.stop_servers()
  312. @skip_if_disabled
  313. def test_cache_remote_image(self):
  314. """
  315. We test that caching is no longer broken for remote images
  316. """
  317. self.cleanup()
  318. self.start_servers(**self.__dict__.copy())
  319. setup_http(self)
  320. # Add a remote image and verify a 201 Created is returned
  321. remote_uri = get_http_uri(self, '2')
  322. headers = {'X-Image-Meta-Name': 'Image2',
  323. 'X-Image-Meta-disk_format': 'raw',
  324. 'X-Image-Meta-container_format': 'ovf',
  325. 'X-Image-Meta-Is-Public': 'True',
  326. 'X-Image-Meta-Location': remote_uri}
  327. path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
  328. http = httplib2.Http()
  329. response, content = http.request(path, 'POST', headers=headers)
  330. self.assertEqual(http_client.CREATED, response.status)
  331. data = jsonutils.loads(content)
  332. self.assertEqual(FIVE_KB, data['image']['size'])
  333. image_id = data['image']['id']
  334. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  335. image_id)
  336. # Grab the image
  337. http = httplib2.Http()
  338. response, content = http.request(path, 'GET')
  339. self.assertEqual(http_client.OK, response.status)
  340. # Grab the image again to ensure it can be served out from
  341. # cache with the correct size
  342. http = httplib2.Http()
  343. response, content = http.request(path, 'GET')
  344. self.assertEqual(http_client.OK, response.status)
  345. self.assertEqual(FIVE_KB, int(response['content-length']))
  346. self.stop_servers()
  347. @skip_if_disabled
  348. def test_cache_middleware_trans_v1_without_download_image_policy(self):
  349. """
  350. Ensure the image v1 API image transfer applied 'download_image'
  351. policy enforcement.
  352. """
  353. self.cleanup()
  354. self.start_servers(**self.__dict__.copy())
  355. # Add an image and verify a 200 OK is returned
  356. image_data = b"*" * FIVE_KB
  357. headers = minimal_headers('Image1')
  358. path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
  359. http = httplib2.Http()
  360. response, content = http.request(path, 'POST', headers=headers,
  361. body=image_data)
  362. self.assertEqual(http_client.CREATED, response.status)
  363. data = jsonutils.loads(content)
  364. self.assertEqual(hashlib.md5(image_data).hexdigest(),
  365. data['image']['checksum'])
  366. self.assertEqual(FIVE_KB, data['image']['size'])
  367. self.assertEqual("Image1", data['image']['name'])
  368. self.assertTrue(data['image']['is_public'])
  369. image_id = data['image']['id']
  370. # Verify image not in cache
  371. image_cached_path = os.path.join(self.api_server.image_cache_dir,
  372. image_id)
  373. self.assertFalse(os.path.exists(image_cached_path))
  374. rules = {"context_is_admin": "role:admin", "default": "",
  375. "download_image": "!"}
  376. self.set_policy_rules(rules)
  377. # Grab the image
  378. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  379. image_id)
  380. http = httplib2.Http()
  381. response, content = http.request(path, 'GET')
  382. self.assertEqual(http_client.FORBIDDEN, response.status)
  383. # Now, we delete the image from the server and verify that
  384. # the image cache no longer contains the deleted image
  385. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  386. image_id)
  387. http = httplib2.Http()
  388. response, content = http.request(path, 'DELETE')
  389. self.assertEqual(http_client.OK, response.status)
  390. self.assertFalse(os.path.exists(image_cached_path))
  391. self.stop_servers()
  392. @skip_if_disabled
  393. def test_cache_middleware_trans_v2_without_download_image_policy(self):
  394. """
  395. Ensure the image v2 API image transfer applied 'download_image'
  396. policy enforcement.
  397. """
  398. self.cleanup()
  399. self.start_servers(**self.__dict__.copy())
  400. # Add an image and verify success
  401. path = "http://%s:%d/v2/images" % ("0.0.0.0", self.api_port)
  402. http = httplib2.Http()
  403. headers = {'content-type': 'application/json'}
  404. image_entity = {
  405. 'name': 'Image1',
  406. 'visibility': 'public',
  407. 'container_format': 'bare',
  408. 'disk_format': 'raw',
  409. }
  410. response, content = http.request(path, 'POST',
  411. headers=headers,
  412. body=jsonutils.dumps(image_entity))
  413. self.assertEqual(http_client.CREATED, response.status)
  414. data = jsonutils.loads(content)
  415. image_id = data['id']
  416. path = "http://%s:%d/v2/images/%s/file" % ("0.0.0.0", self.api_port,
  417. image_id)
  418. headers = {'content-type': 'application/octet-stream'}
  419. image_data = "*" * FIVE_KB
  420. response, content = http.request(path, 'PUT',
  421. headers=headers,
  422. body=image_data)
  423. self.assertEqual(http_client.NO_CONTENT, response.status)
  424. # Verify image not in cache
  425. image_cached_path = os.path.join(self.api_server.image_cache_dir,
  426. image_id)
  427. self.assertFalse(os.path.exists(image_cached_path))
  428. rules = {"context_is_admin": "role:admin", "default": "",
  429. "download_image": "!"}
  430. self.set_policy_rules(rules)
  431. # Grab the image
  432. http = httplib2.Http()
  433. response, content = http.request(path, 'GET')
  434. self.assertEqual(http_client.FORBIDDEN, response.status)
  435. # Now, we delete the image from the server and verify that
  436. # the image cache no longer contains the deleted image
  437. path = "http://%s:%d/v2/images/%s" % ("0.0.0.0", self.api_port,
  438. image_id)
  439. http = httplib2.Http()
  440. response, content = http.request(path, 'DELETE')
  441. self.assertEqual(http_client.NO_CONTENT, response.status)
  442. self.assertFalse(os.path.exists(image_cached_path))
  443. self.stop_servers()
  444. @skip_if_disabled
  445. def test_cache_middleware_trans_with_deactivated_image(self):
  446. """
  447. Ensure the image v1/v2 API image transfer forbids downloading
  448. deactivated images.
  449. Image deactivation is not available in v1. So, we'll deactivate the
  450. image using v2 but test image transfer with both v1 and v2.
  451. """
  452. self.cleanup()
  453. self.start_servers(**self.__dict__.copy())
  454. # Add an image and verify a 200 OK is returned
  455. image_data = b"*" * FIVE_KB
  456. headers = minimal_headers('Image1')
  457. path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
  458. http = httplib2.Http()
  459. response, content = http.request(path, 'POST', headers=headers,
  460. body=image_data)
  461. self.assertEqual(http_client.CREATED, response.status)
  462. data = jsonutils.loads(content)
  463. self.assertEqual(hashlib.md5(image_data).hexdigest(),
  464. data['image']['checksum'])
  465. self.assertEqual(FIVE_KB, data['image']['size'])
  466. self.assertEqual("Image1", data['image']['name'])
  467. self.assertTrue(data['image']['is_public'])
  468. image_id = data['image']['id']
  469. # Grab the image
  470. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  471. image_id)
  472. http = httplib2.Http()
  473. response, content = http.request(path, 'GET')
  474. self.assertEqual(http_client.OK, response.status)
  475. # Verify image in cache
  476. image_cached_path = os.path.join(self.api_server.image_cache_dir,
  477. image_id)
  478. self.assertTrue(os.path.exists(image_cached_path))
  479. # Deactivate the image using v2
  480. path = "http://%s:%d/v2/images/%s/actions/deactivate"
  481. path = path % ("127.0.0.1", self.api_port, image_id)
  482. http = httplib2.Http()
  483. response, content = http.request(path, 'POST')
  484. self.assertEqual(http_client.NO_CONTENT, response.status)
  485. # Download the image with v1. Ensure it is forbidden
  486. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  487. image_id)
  488. http = httplib2.Http()
  489. response, content = http.request(path, 'GET')
  490. self.assertEqual(http_client.FORBIDDEN, response.status)
  491. # Download the image with v2. This succeeds because
  492. # we are in admin context.
  493. path = "http://%s:%d/v2/images/%s/file" % ("127.0.0.1", self.api_port,
  494. image_id)
  495. http = httplib2.Http()
  496. response, content = http.request(path, 'GET')
  497. self.assertEqual(http_client.OK, response.status)
  498. # Reactivate the image using v2
  499. path = "http://%s:%d/v2/images/%s/actions/reactivate"
  500. path = path % ("127.0.0.1", self.api_port, image_id)
  501. http = httplib2.Http()
  502. response, content = http.request(path, 'POST')
  503. self.assertEqual(http_client.NO_CONTENT, response.status)
  504. # Download the image with v1. Ensure it is allowed
  505. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  506. image_id)
  507. http = httplib2.Http()
  508. response, content = http.request(path, 'GET')
  509. self.assertEqual(http_client.OK, response.status)
  510. # Download the image with v2. Ensure it is allowed
  511. path = "http://%s:%d/v2/images/%s/file" % ("127.0.0.1", self.api_port,
  512. image_id)
  513. http = httplib2.Http()
  514. response, content = http.request(path, 'GET')
  515. self.assertEqual(http_client.OK, response.status)
  516. # Now, we delete the image from the server and verify that
  517. # the image cache no longer contains the deleted image
  518. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  519. image_id)
  520. http = httplib2.Http()
  521. response, content = http.request(path, 'DELETE')
  522. self.assertEqual(http_client.OK, response.status)
  523. self.assertFalse(os.path.exists(image_cached_path))
  524. self.stop_servers()
  525. class BaseCacheManageMiddlewareTest(object):
  526. """Base test class for testing cache management middleware"""
  527. def verify_no_images(self):
  528. path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
  529. http = httplib2.Http()
  530. response, content = http.request(path, 'GET')
  531. self.assertEqual(http_client.OK, response.status)
  532. data = jsonutils.loads(content)
  533. self.assertIn('images', data)
  534. self.assertEqual(0, len(data['images']))
  535. def add_image(self, name):
  536. """
  537. Adds an image and returns the newly-added image
  538. identifier
  539. """
  540. image_data = b"*" * FIVE_KB
  541. headers = minimal_headers('%s' % name)
  542. path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
  543. http = httplib2.Http()
  544. response, content = http.request(path, 'POST', headers=headers,
  545. body=image_data)
  546. self.assertEqual(http_client.CREATED, response.status)
  547. data = jsonutils.loads(content)
  548. self.assertEqual(hashlib.md5(image_data).hexdigest(),
  549. data['image']['checksum'])
  550. self.assertEqual(FIVE_KB, data['image']['size'])
  551. self.assertEqual(name, data['image']['name'])
  552. self.assertTrue(data['image']['is_public'])
  553. return data['image']['id']
  554. def verify_no_cached_images(self):
  555. """
  556. Verify no images in the image cache
  557. """
  558. path = "http://%s:%d/v1/cached_images" % ("127.0.0.1", self.api_port)
  559. http = httplib2.Http()
  560. response, content = http.request(path, 'GET')
  561. self.assertEqual(http_client.OK, response.status)
  562. data = jsonutils.loads(content)
  563. self.assertIn('cached_images', data)
  564. self.assertEqual([], data['cached_images'])
  565. @skip_if_disabled
  566. def test_user_not_authorized(self):
  567. self.cleanup()
  568. self.start_servers(**self.__dict__.copy())
  569. self.verify_no_images()
  570. image_id1 = self.add_image("Image1")
  571. image_id2 = self.add_image("Image2")
  572. # Verify image does not yet show up in cache (we haven't "hit"
  573. # it yet using a GET /images/1 ...
  574. self.verify_no_cached_images()
  575. # Grab the image
  576. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  577. image_id1)
  578. http = httplib2.Http()
  579. response, content = http.request(path, 'GET')
  580. self.assertEqual(http_client.OK, response.status)
  581. # Verify image now in cache
  582. path = "http://%s:%d/v1/cached_images" % ("127.0.0.1", self.api_port)
  583. http = httplib2.Http()
  584. response, content = http.request(path, 'GET')
  585. self.assertEqual(http_client.OK, response.status)
  586. data = jsonutils.loads(content)
  587. self.assertIn('cached_images', data)
  588. cached_images = data['cached_images']
  589. self.assertEqual(1, len(cached_images))
  590. self.assertEqual(image_id1, cached_images[0]['image_id'])
  591. # Set policy to disallow access to cache management
  592. rules = {"manage_image_cache": '!'}
  593. self.set_policy_rules(rules)
  594. # Verify an unprivileged user cannot see cached images
  595. path = "http://%s:%d/v1/cached_images" % ("127.0.0.1", self.api_port)
  596. http = httplib2.Http()
  597. response, content = http.request(path, 'GET')
  598. self.assertEqual(http_client.FORBIDDEN, response.status)
  599. # Verify an unprivileged user cannot delete images from the cache
  600. path = "http://%s:%d/v1/cached_images/%s" % ("127.0.0.1",
  601. self.api_port, image_id1)
  602. http = httplib2.Http()
  603. response, content = http.request(path, 'DELETE')
  604. self.assertEqual(http_client.FORBIDDEN, response.status)
  605. # Verify an unprivileged user cannot delete all cached images
  606. path = "http://%s:%d/v1/cached_images" % ("127.0.0.1", self.api_port)
  607. http = httplib2.Http()
  608. response, content = http.request(path, 'DELETE')
  609. self.assertEqual(http_client.FORBIDDEN, response.status)
  610. # Verify an unprivileged user cannot queue an image
  611. path = "http://%s:%d/v1/queued_images/%s" % ("127.0.0.1",
  612. self.api_port, image_id2)
  613. http = httplib2.Http()
  614. response, content = http.request(path, 'PUT')
  615. self.assertEqual(http_client.FORBIDDEN, response.status)
  616. self.stop_servers()
  617. @skip_if_disabled
  618. def test_cache_manage_get_cached_images(self):
  619. """
  620. Tests that cached images are queryable
  621. """
  622. self.cleanup()
  623. self.start_servers(**self.__dict__.copy())
  624. self.verify_no_images()
  625. image_id = self.add_image("Image1")
  626. # Verify image does not yet show up in cache (we haven't "hit"
  627. # it yet using a GET /images/1 ...
  628. self.verify_no_cached_images()
  629. # Grab the image
  630. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  631. image_id)
  632. http = httplib2.Http()
  633. response, content = http.request(path, 'GET')
  634. self.assertEqual(http_client.OK, response.status)
  635. # Verify image now in cache
  636. path = "http://%s:%d/v1/cached_images" % ("127.0.0.1", self.api_port)
  637. http = httplib2.Http()
  638. response, content = http.request(path, 'GET')
  639. self.assertEqual(http_client.OK, response.status)
  640. data = jsonutils.loads(content)
  641. self.assertIn('cached_images', data)
  642. # Verify the last_modified/last_accessed values are valid floats
  643. for cached_image in data['cached_images']:
  644. for time_key in ('last_modified', 'last_accessed'):
  645. time_val = cached_image[time_key]
  646. try:
  647. float(time_val)
  648. except ValueError:
  649. self.fail('%s time %s for cached image %s not a valid '
  650. 'float' % (time_key, time_val,
  651. cached_image['image_id']))
  652. cached_images = data['cached_images']
  653. self.assertEqual(1, len(cached_images))
  654. self.assertEqual(image_id, cached_images[0]['image_id'])
  655. self.assertEqual(0, cached_images[0]['hits'])
  656. # Hit the image
  657. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  658. image_id)
  659. http = httplib2.Http()
  660. response, content = http.request(path, 'GET')
  661. self.assertEqual(http_client.OK, response.status)
  662. # Verify image hits increased in output of manage GET
  663. path = "http://%s:%d/v1/cached_images" % ("127.0.0.1", self.api_port)
  664. http = httplib2.Http()
  665. response, content = http.request(path, 'GET')
  666. self.assertEqual(http_client.OK, response.status)
  667. data = jsonutils.loads(content)
  668. self.assertIn('cached_images', data)
  669. cached_images = data['cached_images']
  670. self.assertEqual(1, len(cached_images))
  671. self.assertEqual(image_id, cached_images[0]['image_id'])
  672. self.assertEqual(1, cached_images[0]['hits'])
  673. self.stop_servers()
  674. @skip_if_disabled
  675. def test_cache_manage_delete_cached_images(self):
  676. """
  677. Tests that cached images may be deleted
  678. """
  679. self.cleanup()
  680. self.start_servers(**self.__dict__.copy())
  681. self.verify_no_images()
  682. ids = {}
  683. # Add a bunch of images...
  684. for x in range(4):
  685. ids[x] = self.add_image("Image%s" % str(x))
  686. # Verify no images in cached_images because no image has been hit
  687. # yet using a GET /images/<IMAGE_ID> ...
  688. self.verify_no_cached_images()
  689. # Grab the images, essentially caching them...
  690. for x in range(4):
  691. path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
  692. ids[x])
  693. http = httplib2.Http()
  694. response, content = http.request(path, 'GET')
  695. self.assertEqual(http_client.OK, response.status,
  696. "Failed to find image %s" % ids[x])
  697. # Verify images now in cache
  698. path = "http://%s:%d/v1/cached_images" % ("127.0.0.1", self.api_port)
  699. http = httplib2.Http()
  700. response, content = http.request(path, 'GET')
  701. self.assertEqual(http_client.OK, response.status)
  702. data = jsonutils.loads(content)
  703. self.assertIn('cached_images', data)
  704. cached_images = data['cached_images']
  705. self.assertEqual(4, len(cached_images))
  706. for x in range(4, 0): # Cached images returned last modified order
  707. self.assertEqual(ids[x], cached_images[x]['image_id'])
  708. self.assertEqual(0, cached_images[x]['hits'])
  709. # Delete third image of the cached images and verify no longer in cache
  710. path = "http://%s:%d/v1/cached_images/%s" % ("127.0.0.1",
  711. self.api_port, ids[2])
  712. http = httplib2.Http()
  713. response, content = http.request(path, 'DELETE')
  714. self.assertEqual(http_client.OK, response.status)
  715. path = "http://%s:%d/v1/cached_images" % ("127.0.0.1", self.api_port)
  716. http = httplib2.Http()
  717. response, content = http.request(path, 'GET')
  718. self.assertEqual(http_client.OK, response.status)
  719. data = jsonutils.loads(content)
  720. self.assertIn('cached_images', data)
  721. cached_images = data['cached_images']
  722. self.assertEqual(3, len(cached_images))
  723. self.assertNotIn(ids[2], [x['image_id'] for x in cached_images])
  724. # Delete all cached images and verify nothing in cache
  725. path = "http://%s:%d/v1/cached_images" % ("127.0.0.1", self.api_port)
  726. http = httplib2.Http()
  727. response, content = http.request(path, 'DELETE')
  728. self.assertEqual(http_client.OK, response.status)
  729. path = "http://%s:%d/v1/cached_images" % ("127.0.0.1", self.api_port)
  730. http = httplib2.Http()
  731. response, content = http.request(path, 'GET')
  732. self.assertEqual(http_client.OK, response.status)
  733. data = jsonutils.loads(content)
  734. self.assertIn('cached_images', data)
  735. cached_images = data['cached_images']
  736. self.assertEqual(0, len(cached_images))
  737. self.stop_servers()
  738. @skip_if_disabled
  739. def test_cache_manage_delete_queued_images(self):
  740. """
  741. Tests that all queued images may be deleted at once
  742. """
  743. self.cleanup()
  744. self.start_servers(**self.__dict__.copy())
  745. self.verify_no_images()
  746. ids = {}
  747. NUM_IMAGES = 4
  748. # Add and then queue some images
  749. for x in range(NUM_IMAGES):
  750. ids[x] = self.add_image("Image%s" % str(x))
  751. path = "http://%s:%d/v1/queued_images/%s" % ("127.0.0.1",
  752. self.api_port, ids[x])
  753. http = httplib2.Http()
  754. response, content = http.request(path, 'PUT')
  755. self.assertEqual(http_client.OK, response.status)
  756. # Delete all queued images
  757. path = "http://%s:%d/v1/queued_images" % ("127.0.0.1", self.api_port)
  758. http = httplib2.Http()
  759. response, content = http.request(path, 'DELETE')
  760. self.assertEqual(http_client.OK, response.status)
  761. data = jsonutils.loads(content)
  762. num_deleted = data['num_deleted']
  763. self.assertEqual(NUM_IMAGES, num_deleted)
  764. # Verify a second delete now returns num_deleted=0
  765. path = "http://%s:%d/v1/queued_images" % ("127.0.0.1", self.api_port)
  766. http = httplib2.Http()
  767. response, content = http.request(path, 'DELETE')
  768. self.assertEqual(http_client.OK, response.status)
  769. data = jsonutils.loads(content)
  770. num_deleted = data['num_deleted']
  771. self.assertEqual(0, num_deleted)
  772. self.stop_servers()
  773. @skip_if_disabled
  774. def test_queue_and_prefetch(self):
  775. """
  776. Tests that images may be queued and prefetched
  777. """
  778. self.cleanup()
  779. self.start_servers(**self.__dict__.copy())
  780. cache_config_filepath = os.path.join(self.test_dir, 'etc',
  781. 'glance-cache.conf')
  782. cache_file_options = {
  783. 'image_cache_dir': self.api_server.image_cache_dir,
  784. 'image_cache_driver': self.image_cache_driver,
  785. 'registry_port': self.registry_server.bind_port,
  786. 'log_file': os.path.join(self.test_dir, 'cache.log'),
  787. 'lock_path': self.test_dir,
  788. 'metadata_encryption_key': "012345678901234567890123456789ab",
  789. 'filesystem_store_datadir': self.test_dir
  790. }
  791. with open(cache_config_filepath, 'w') as cache_file:
  792. cache_file.write("""[DEFAULT]
  793. debug = True
  794. lock_path = %(lock_path)s
  795. image_cache_dir = %(image_cache_dir)s
  796. image_cache_driver = %(image_cache_driver)s
  797. registry_host = 127.0.0.1
  798. registry_port = %(registry_port)s
  799. metadata_encryption_key = %(metadata_encryption_key)s
  800. log_file = %(log_file)s
  801. [glance_store]
  802. filesystem_store_datadir=%(filesystem_store_datadir)s
  803. """ % cache_file_options)
  804. self.verify_no_images()
  805. ids = {}
  806. # Add a bunch of images...
  807. for x in range(4):
  808. ids[x] = self.add_image("Image%s" % str(x))
  809. # Queue the first image, verify no images still in cache after queueing
  810. # then run the prefetcher and verify that the image is then in the
  811. # cache
  812. path = "http://%s:%d/v1/queued_images/%s" % ("127.0.0.1",
  813. self.api_port, ids[0])
  814. http = httplib2.Http()
  815. response, content = http.request(path, 'PUT')
  816. self.assertEqual(http_client.OK, response.status)
  817. self.verify_no_cached_images()
  818. cmd = ("%s -m glance.cmd.cache_prefetcher --config-file %s" %
  819. (sys.executable, cache_config_filepath))
  820. exitcode, out, err = execute(cmd)
  821. self.assertEqual(0, exitcode)
  822. self.assertEqual(b'', out.strip(), out)
  823. # Verify first image now in cache
  824. path = "http://%s:%d/v1/cached_images" % ("127.0.0.1", self.api_port)
  825. http = httplib2.Http()
  826. response, content = http.request(path, 'GET')
  827. self.assertEqual(http_client.OK, response.status)
  828. data = jsonutils.loads(content)
  829. self.assertIn('cached_images', data)
  830. cached_images = data['cached_images']
  831. self.assertEqual(1, len(cached_images))
  832. self.assertIn(ids[0], [r['image_id']
  833. for r in data['cached_images']])
  834. self.stop_servers()
  835. class TestImageCacheXattr(functional.FunctionalTest,
  836. BaseCacheMiddlewareTest):
  837. """Functional tests that exercise the image cache using the xattr driver"""
  838. def setUp(self):
  839. """
  840. Test to see if the pre-requisites for the image cache
  841. are working (python-xattr installed and xattr support on the
  842. filesystem)
  843. """
  844. if getattr(self, 'disabled', False):
  845. return
  846. if not getattr(self, 'inited', False):
  847. try:
  848. import xattr # noqa
  849. except ImportError:
  850. self.inited = True
  851. self.disabled = True
  852. self.disabled_message = ("python-xattr not installed.")
  853. return
  854. self.inited = True
  855. self.disabled = False
  856. self.image_cache_driver = "xattr"
  857. super(TestImageCacheXattr, self).setUp()
  858. self.api_server.deployment_flavor = "caching"
  859. if not xattr_writes_supported(self.test_dir):
  860. self.inited = True
  861. self.disabled = True
  862. self.disabled_message = ("filesystem does not support xattr")
  863. return
  864. def tearDown(self):
  865. super(TestImageCacheXattr, self).tearDown()
  866. if os.path.exists(self.api_server.image_cache_dir):
  867. shutil.rmtree(self.api_server.image_cache_dir)
  868. class TestImageCacheManageXattr(functional.FunctionalTest,
  869. BaseCacheManageMiddlewareTest):
  870. """
  871. Functional tests that exercise the image cache management
  872. with the Xattr cache driver
  873. """
  874. def setUp(self):
  875. """
  876. Test to see if the pre-requisites for the image cache
  877. are working (python-xattr installed and xattr support on the
  878. filesystem)
  879. """
  880. if getattr(self, 'disabled', False):
  881. return
  882. if not getattr(self, 'inited', False):
  883. try:
  884. import xattr # noqa
  885. except ImportError:
  886. self.inited = True
  887. self.disabled = True
  888. self.disabled_message = ("python-xattr not installed.")
  889. return
  890. self.inited = True
  891. self.disabled = False
  892. self.image_cache_driver = "xattr"
  893. super(TestImageCacheManageXattr, self).setUp()
  894. self.api_server.deployment_flavor = "cachemanagement"
  895. if not xattr_writes_supported(self.test_dir):
  896. self.inited = True
  897. self.disabled = True
  898. self.disabled_message = ("filesystem does not support xattr")
  899. return
  900. def tearDown(self):
  901. super(TestImageCacheManageXattr, self).tearDown()
  902. if os.path.exists(self.api_server.image_cache_dir):
  903. shutil.rmtree(self.api_server.image_cache_dir)
  904. class TestImageCacheSqlite(functional.FunctionalTest,
  905. BaseCacheMiddlewareTest):
  906. """
  907. Functional tests that exercise the image cache using the
  908. SQLite driver
  909. """
  910. def setUp(self):
  911. """
  912. Test to see if the pre-requisites for the image cache
  913. are working (python-xattr installed and xattr support on the
  914. filesystem)
  915. """
  916. if getattr(self, 'disabled', False):
  917. return
  918. if not getattr(self, 'inited', False):
  919. try:
  920. import sqlite3 # noqa
  921. except ImportError:
  922. self.inited = True
  923. self.disabled = True
  924. self.disabled_message = ("python-sqlite3 not installed.")
  925. return
  926. self.inited = True
  927. self.disabled = False
  928. super(TestImageCacheSqlite, self).setUp()
  929. self.api_server.deployment_flavor = "caching"
  930. def tearDown(self):
  931. super(TestImageCacheSqlite, self).tearDown()
  932. if os.path.exists(self.api_server.image_cache_dir):
  933. shutil.rmtree(self.api_server.image_cache_dir)
  934. class TestImageCacheManageSqlite(functional.FunctionalTest,
  935. BaseCacheManageMiddlewareTest):
  936. """
  937. Functional tests that exercise the image cache management using the
  938. SQLite driver
  939. """
  940. def setUp(self):
  941. """
  942. Test to see if the pre-requisites for the image cache
  943. are working (python-xattr installed and xattr support on the
  944. filesystem)
  945. """
  946. if getattr(self, 'disabled', False):
  947. return
  948. if not getattr(self, 'inited', False):
  949. try:
  950. import sqlite3 # noqa
  951. except ImportError:
  952. self.inited = True
  953. self.disabled = True
  954. self.disabled_message = ("python-sqlite3 not installed.")
  955. return
  956. self.inited = True
  957. self.disabled = False
  958. self.image_cache_driver = "sqlite"
  959. super(TestImageCacheManageSqlite, self).setUp()
  960. self.api_server.deployment_flavor = "cachemanagement"
  961. def tearDown(self):
  962. super(TestImageCacheManageSqlite, self).tearDown()
  963. if os.path.exists(self.api_server.image_cache_dir):
  964. shutil.rmtree(self.api_server.image_cache_dir)