OpenStack Image Management (Glance)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

test_scrubber.py 16KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387
  1. # Copyright 2011-2012 OpenStack Foundation
  2. # All Rights Reserved.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. import os
  16. import sys
  17. import time
  18. import httplib2
  19. from oslo_config import cfg
  20. from oslo_serialization import jsonutils
  21. from six.moves import http_client
  22. # NOTE(jokke): simplified transition to py3, behaves like py2 xrange
  23. from six.moves import range
  24. from glance import context
  25. import glance.db as db_api
  26. from glance.tests import functional
  27. from glance.tests.utils import execute
  28. CONF = cfg.CONF
  29. class TestScrubber(functional.FunctionalTest):
  30. """Test that delayed_delete works and the scrubber deletes"""
  31. def setUp(self):
  32. super(TestScrubber, self).setUp()
  33. self.admin_context = context.get_admin_context(show_deleted=True)
  34. CONF.set_override('sql_connection', self.api_server.sql_connection)
  35. def _send_create_image_http_request(self, path, body=None):
  36. headers = {
  37. "Content-Type": "application/json"
  38. }
  39. body = body or {'container_format': 'ovf',
  40. 'disk_format': 'raw',
  41. 'name': 'test_image',
  42. 'visibility': 'public'}
  43. body = jsonutils.dumps(body)
  44. return httplib2.Http().request(path, 'POST', body, headers)
  45. def _send_upload_image_http_request(self, path, body=None):
  46. headers = {
  47. "Content-Type": "application/octet-stream"
  48. }
  49. return httplib2.Http().request(path, 'PUT', body, headers)
  50. def _send_http_request(self, path, method):
  51. headers = {
  52. "Content-Type": "application/json"
  53. }
  54. return httplib2.Http().request(path, method, None, headers)
  55. def _get_pending_delete_image(self, image_id):
  56. # In Glance V2, there is no way to get the 'pending_delete' image from
  57. # API. So we get the image from db here for testing.
  58. # Clean the session cache first to avoid connecting to the old db data.
  59. db_api.get_api()._FACADE = None
  60. image = db_api.get_api().image_get(self.admin_context, image_id)
  61. return image
  62. def test_delayed_delete(self):
  63. """
  64. test that images don't get deleted immediately and that the scrubber
  65. scrubs them
  66. """
  67. self.cleanup()
  68. self.start_servers(delayed_delete=True, daemon=True,
  69. metadata_encryption_key='')
  70. path = "http://%s:%d/v2/images" % ("127.0.0.1", self.api_port)
  71. response, content = self._send_create_image_http_request(path)
  72. self.assertEqual(http_client.CREATED, response.status)
  73. image = jsonutils.loads(content)
  74. self.assertEqual('queued', image['status'])
  75. file_path = "%s/%s/file" % (path, image['id'])
  76. response, content = self._send_upload_image_http_request(file_path,
  77. body='XXX')
  78. self.assertEqual(http_client.NO_CONTENT, response.status)
  79. path = "%s/%s" % (path, image['id'])
  80. response, content = self._send_http_request(path, 'GET')
  81. image = jsonutils.loads(content)
  82. self.assertEqual('active', image['status'])
  83. response, content = self._send_http_request(path, 'DELETE')
  84. self.assertEqual(http_client.NO_CONTENT, response.status)
  85. image = self._get_pending_delete_image(image['id'])
  86. self.assertEqual('pending_delete', image['status'])
  87. self.wait_for_scrub(image['id'])
  88. self.stop_servers()
  89. def test_scrubber_app(self):
  90. """
  91. test that the glance-scrubber script runs successfully when not in
  92. daemon mode
  93. """
  94. self.cleanup()
  95. self.start_servers(delayed_delete=True, daemon=False,
  96. metadata_encryption_key='')
  97. path = "http://%s:%d/v2/images" % ("127.0.0.1", self.api_port)
  98. response, content = self._send_create_image_http_request(path)
  99. self.assertEqual(http_client.CREATED, response.status)
  100. image = jsonutils.loads(content)
  101. self.assertEqual('queued', image['status'])
  102. file_path = "%s/%s/file" % (path, image['id'])
  103. response, content = self._send_upload_image_http_request(file_path,
  104. body='XXX')
  105. self.assertEqual(http_client.NO_CONTENT, response.status)
  106. path = "%s/%s" % (path, image['id'])
  107. response, content = self._send_http_request(path, 'GET')
  108. image = jsonutils.loads(content)
  109. self.assertEqual('active', image['status'])
  110. response, content = self._send_http_request(path, 'DELETE')
  111. self.assertEqual(http_client.NO_CONTENT, response.status)
  112. image = self._get_pending_delete_image(image['id'])
  113. self.assertEqual('pending_delete', image['status'])
  114. # wait for the scrub time on the image to pass
  115. time.sleep(self.api_server.scrub_time)
  116. # scrub images and make sure they get deleted
  117. exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable
  118. cmd = ("%s --config-file %s" %
  119. (exe_cmd, self.scrubber_daemon.conf_file_name))
  120. exitcode, out, err = execute(cmd, raise_error=False)
  121. self.assertEqual(0, exitcode)
  122. self.wait_for_scrub(image['id'])
  123. self.stop_servers()
  124. def test_scrubber_delete_handles_exception(self):
  125. """
  126. Test that the scrubber handles the case where an
  127. exception occurs when _delete() is called. The scrubber
  128. should not write out queue files in this case.
  129. """
  130. # Start servers.
  131. self.cleanup()
  132. self.start_servers(delayed_delete=True, daemon=False,
  133. default_store='file')
  134. # Check that we are using a file backend.
  135. self.assertEqual(self.api_server.default_store, 'file')
  136. # add an image
  137. path = "http://%s:%d/v2/images" % ("127.0.0.1", self.api_port)
  138. response, content = self._send_create_image_http_request(path)
  139. self.assertEqual(http_client.CREATED, response.status)
  140. image = jsonutils.loads(content)
  141. self.assertEqual('queued', image['status'])
  142. file_path = "%s/%s/file" % (path, image['id'])
  143. response, content = self._send_upload_image_http_request(file_path,
  144. body='XXX')
  145. self.assertEqual(http_client.NO_CONTENT, response.status)
  146. path = "%s/%s" % (path, image['id'])
  147. response, content = self._send_http_request(path, 'GET')
  148. image = jsonutils.loads(content)
  149. self.assertEqual('active', image['status'])
  150. # delete the image
  151. response, content = self._send_http_request(path, 'DELETE')
  152. self.assertEqual(http_client.NO_CONTENT, response.status)
  153. # ensure the image is marked pending delete.
  154. image = self._get_pending_delete_image(image['id'])
  155. self.assertEqual('pending_delete', image['status'])
  156. # Remove the file from the backend.
  157. file_path = os.path.join(self.api_server.image_dir, image['id'])
  158. os.remove(file_path)
  159. # Wait for the scrub time on the image to pass
  160. time.sleep(self.api_server.scrub_time)
  161. # run the scrubber app, and ensure it doesn't fall over
  162. exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable
  163. cmd = ("%s --config-file %s" %
  164. (exe_cmd, self.scrubber_daemon.conf_file_name))
  165. exitcode, out, err = execute(cmd, raise_error=False)
  166. self.assertEqual(0, exitcode)
  167. self.wait_for_scrub(image['id'])
  168. self.stop_servers()
  169. def test_scrubber_app_queue_errors_not_daemon(self):
  170. """
  171. test that the glance-scrubber exits with an exit code > 0 when it
  172. fails to lookup images, indicating a configuration error when not
  173. in daemon mode.
  174. Related-Bug: #1548289
  175. """
  176. # Don't start the registry server to cause intended failure
  177. # Don't start the api server to save time
  178. exitcode, out, err = self.scrubber_daemon.start(
  179. delayed_delete=True, daemon=False)
  180. self.assertEqual(0, exitcode,
  181. "Failed to spin up the Scrubber daemon. "
  182. "Got: %s" % err)
  183. # Run the Scrubber
  184. exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable
  185. cmd = ("%s --config-file %s" %
  186. (exe_cmd, self.scrubber_daemon.conf_file_name))
  187. exitcode, out, err = execute(cmd, raise_error=False)
  188. self.assertEqual(1, exitcode)
  189. self.assertIn('Can not get scrub jobs from queue', str(err))
  190. self.stop_server(self.scrubber_daemon)
  191. def test_scrubber_restore_image(self):
  192. self.cleanup()
  193. self.start_servers(delayed_delete=True, daemon=False,
  194. metadata_encryption_key='')
  195. path = "http://%s:%d/v2/images" % ("127.0.0.1", self.api_port)
  196. response, content = self._send_create_image_http_request(path)
  197. self.assertEqual(http_client.CREATED, response.status)
  198. image = jsonutils.loads(content)
  199. self.assertEqual('queued', image['status'])
  200. file_path = "%s/%s/file" % (path, image['id'])
  201. response, content = self._send_upload_image_http_request(file_path,
  202. body='XXX')
  203. self.assertEqual(http_client.NO_CONTENT, response.status)
  204. path = "%s/%s" % (path, image['id'])
  205. response, content = self._send_http_request(path, 'GET')
  206. image = jsonutils.loads(content)
  207. self.assertEqual('active', image['status'])
  208. response, content = self._send_http_request(path, 'DELETE')
  209. self.assertEqual(http_client.NO_CONTENT, response.status)
  210. image = self._get_pending_delete_image(image['id'])
  211. self.assertEqual('pending_delete', image['status'])
  212. def _test_content():
  213. exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable
  214. cmd = ("%s --config-file %s --restore %s" %
  215. (exe_cmd, self.scrubber_daemon.conf_file_name, image['id']))
  216. return execute(cmd, raise_error=False)
  217. exitcode, out, err = self.wait_for_scrubber_shutdown(_test_content)
  218. self.assertEqual(0, exitcode)
  219. response, content = self._send_http_request(path, 'GET')
  220. image = jsonutils.loads(content)
  221. self.assertEqual('active', image['status'])
  222. self.stop_servers()
  223. def test_scrubber_restore_active_image_raise_error(self):
  224. self.cleanup()
  225. self.start_servers(delayed_delete=True, daemon=False,
  226. metadata_encryption_key='')
  227. path = "http://%s:%d/v2/images" % ("127.0.0.1", self.api_port)
  228. response, content = self._send_create_image_http_request(path)
  229. self.assertEqual(http_client.CREATED, response.status)
  230. image = jsonutils.loads(content)
  231. self.assertEqual('queued', image['status'])
  232. file_path = "%s/%s/file" % (path, image['id'])
  233. response, content = self._send_upload_image_http_request(file_path,
  234. body='XXX')
  235. self.assertEqual(http_client.NO_CONTENT, response.status)
  236. path = "%s/%s" % (path, image['id'])
  237. response, content = self._send_http_request(path, 'GET')
  238. image = jsonutils.loads(content)
  239. self.assertEqual('active', image['status'])
  240. def _test_content():
  241. exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable
  242. cmd = ("%s --config-file %s --restore %s" %
  243. (exe_cmd, self.scrubber_daemon.conf_file_name, image['id']))
  244. return execute(cmd, raise_error=False)
  245. exitcode, out, err = self.wait_for_scrubber_shutdown(_test_content)
  246. self.assertEqual(1, exitcode)
  247. self.assertIn('cannot restore the image from active to active '
  248. '(wanted from_state=pending_delete)', str(err))
  249. self.stop_servers()
  250. def test_scrubber_restore_image_non_exist(self):
  251. def _test_content():
  252. scrubber = functional.ScrubberDaemon(self.test_dir,
  253. self.policy_file)
  254. scrubber.write_conf(daemon=False)
  255. scrubber.needs_database = True
  256. scrubber.create_database()
  257. exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable
  258. cmd = ("%s --config-file %s --restore fake_image_id" %
  259. (exe_cmd, scrubber.conf_file_name))
  260. return execute(cmd, raise_error=False)
  261. exitcode, out, err = self.wait_for_scrubber_shutdown(_test_content)
  262. self.assertEqual(1, exitcode)
  263. self.assertIn('No image found with ID fake_image_id', str(err))
  264. def test_scrubber_restore_image_with_daemon_raise_error(self):
  265. exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable
  266. cmd = ("%s --daemon --restore fake_image_id" % exe_cmd)
  267. exitcode, out, err = execute(cmd, raise_error=False)
  268. self.assertEqual(1, exitcode)
  269. self.assertIn('The restore and daemon options should not be set '
  270. 'together', str(err))
  271. def test_scrubber_restore_image_with_daemon_running(self):
  272. self.cleanup()
  273. self.scrubber_daemon.start(daemon=True)
  274. exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable
  275. cmd = ("%s --restore fake_image_id" % exe_cmd)
  276. exitcode, out, err = execute(cmd, raise_error=False)
  277. self.assertEqual(1, exitcode)
  278. self.assertIn('The glance-scrubber process is running under daemon',
  279. str(err))
  280. self.stop_server(self.scrubber_daemon)
  281. def wait_for_scrubber_shutdown(self, func):
  282. # NOTE(wangxiyuan, rosmaita): The image-restore functionality contains
  283. # a check to make sure the scrubber isn't also running in daemon mode
  284. # to prevent a race condition between a delete and a restore.
  285. # Sometimes the glance-scrubber process which is setup by the
  286. # previous test can't be shutdown immediately, so if we get the "daemon
  287. # running" message we sleep and try again.
  288. not_down_msg = 'The glance-scrubber process is running under daemon'
  289. total_wait = 15
  290. for _ in range(total_wait):
  291. exitcode, out, err = func()
  292. if exitcode == 1 and not_down_msg in str(err):
  293. time.sleep(1)
  294. continue
  295. return exitcode, out, err
  296. else:
  297. self.fail('Scrubber did not shut down within {} sec'.format(
  298. total_wait))
  299. def wait_for_scrub(self, image_id):
  300. """
  301. NOTE(jkoelker) The build servers sometimes take longer than 15 seconds
  302. to scrub. Give it up to 5 min, checking checking every 15 seconds.
  303. When/if it flips to deleted, bail immediately.
  304. """
  305. wait_for = 300 # seconds
  306. check_every = 15 # seconds
  307. for _ in range(wait_for // check_every):
  308. time.sleep(check_every)
  309. image = db_api.get_api().image_get(self.admin_context, image_id)
  310. if (image['status'] == 'deleted' and
  311. image['deleted'] == True):
  312. break
  313. else:
  314. continue
  315. else:
  316. self.fail('image was never scrubbed')