Manage a pool of nodes for a distributed test infrastructure
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

test_builder.py 15KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346
  1. # Copyright (C) 2015 Hewlett-Packard Development Company, L.P.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12. # implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. import os
  16. import uuid
  17. import fixtures
  18. import mock
  19. import subprocess
  20. from nodepool import builder, exceptions, tests
  21. from nodepool.driver.fake import provider as fakeprovider
  22. from nodepool import zk
  23. class TestNodepoolBuilderDibImage(tests.BaseTestCase):
  24. def test_from_path(self):
  25. image = builder.DibImageFile.from_path(
  26. '/foo/bar/myid1234.qcow2')
  27. self.assertEqual(image.image_id, 'myid1234')
  28. self.assertEqual(image.extension, 'qcow2')
  29. def test_from_image_id(self):
  30. tempdir = fixtures.TempDir()
  31. self.useFixture(tempdir)
  32. image_path = os.path.join(tempdir.path, 'myid1234.qcow2')
  33. open(image_path, 'w')
  34. images = builder.DibImageFile.from_image_id(tempdir.path, 'myid1234')
  35. self.assertEqual(len(images), 1)
  36. image = images[0]
  37. self.assertEqual(image.image_id, 'myid1234')
  38. self.assertEqual(image.extension, 'qcow2')
  39. def test_from_id_multiple(self):
  40. tempdir = fixtures.TempDir()
  41. self.useFixture(tempdir)
  42. image_path_1 = os.path.join(tempdir.path, 'myid1234.qcow2')
  43. image_path_2 = os.path.join(tempdir.path, 'myid1234.raw')
  44. open(image_path_1, 'w')
  45. open(image_path_2, 'w')
  46. images = builder.DibImageFile.from_image_id(tempdir.path, 'myid1234')
  47. images = sorted(images, key=lambda x: x.extension)
  48. self.assertEqual(len(images), 2)
  49. self.assertEqual(images[0].extension, 'qcow2')
  50. self.assertEqual(images[1].extension, 'raw')
  51. def test_from_images_dir(self):
  52. tempdir = fixtures.TempDir()
  53. self.useFixture(tempdir)
  54. image_path_1 = os.path.join(tempdir.path, 'myid1234.qcow2')
  55. image_path_2 = os.path.join(tempdir.path, 'myid1234.raw')
  56. open(image_path_1, 'w')
  57. open(image_path_2, 'w')
  58. images = builder.DibImageFile.from_images_dir(tempdir.path)
  59. images = sorted(images, key=lambda x: x.extension)
  60. self.assertEqual(len(images), 2)
  61. self.assertEqual(images[0].image_id, 'myid1234')
  62. self.assertEqual(images[0].extension, 'qcow2')
  63. self.assertEqual(images[1].image_id, 'myid1234')
  64. self.assertEqual(images[1].extension, 'raw')
  65. def test_to_path(self):
  66. image = builder.DibImageFile('myid1234', 'qcow2')
  67. self.assertEqual(image.to_path('/imagedir'),
  68. '/imagedir/myid1234.qcow2')
  69. self.assertEqual(image.to_path('/imagedir/'),
  70. '/imagedir/myid1234.qcow2')
  71. self.assertEqual(image.to_path('/imagedir/', False),
  72. '/imagedir/myid1234')
  73. image = builder.DibImageFile('myid1234')
  74. self.assertRaises(exceptions.BuilderError, image.to_path, '/imagedir/')
  75. class TestNodePoolBuilder(tests.DBTestCase):
  76. def test_start_stop(self):
  77. config = self.setup_config('node.yaml')
  78. nb = builder.NodePoolBuilder(config)
  79. nb.cleanup_interval = .5
  80. nb.build_interval = .1
  81. nb.upload_interval = .1
  82. nb.start()
  83. nb.stop()
  84. def test_builder_id_file(self):
  85. configfile = self.setup_config('node.yaml')
  86. self.useBuilder(configfile)
  87. path = os.path.join(self._config_images_dir.path, 'builder_id.txt')
  88. # Validate the unique ID file exists and contents are what we expect
  89. self.assertTrue(os.path.exists(path))
  90. with open(path, "r") as f:
  91. the_id = f.read()
  92. obj = uuid.UUID(the_id, version=4)
  93. self.assertEqual(the_id, str(obj))
  94. def test_image_upload_fail(self):
  95. """Test that image upload fails are handled properly."""
  96. # Now swap out the upload fake so that the next uploads fail
  97. fake_client = fakeprovider.FakeUploadFailCloud(times_to_fail=1)
  98. def get_fake_client(*args, **kwargs):
  99. return fake_client
  100. self.useFixture(fixtures.MockPatchObject(
  101. fakeprovider.FakeProvider, '_getClient',
  102. get_fake_client))
  103. configfile = self.setup_config('node.yaml')
  104. pool = self.useNodepool(configfile, watermark_sleep=1)
  105. # NOTE(pabelanger): Disable CleanupWorker thread for nodepool-builder
  106. # as we currently race it to validate our failed uploads.
  107. self.useBuilder(configfile, cleanup_interval=0)
  108. pool.start()
  109. self.waitForImage('fake-provider', 'fake-image')
  110. nodes = self.waitForNodes('fake-label')
  111. self.assertEqual(len(nodes), 1)
  112. newest_builds = self.zk.getMostRecentBuilds(1, 'fake-image',
  113. state=zk.READY)
  114. self.assertEqual(1, len(newest_builds))
  115. uploads = self.zk.getUploads('fake-image', newest_builds[0].id,
  116. 'fake-provider', states=[zk.FAILED])
  117. self.assertEqual(1, len(uploads))
  118. def test_provider_addition(self):
  119. configfile = self.setup_config('node.yaml')
  120. self.useBuilder(configfile)
  121. self.waitForImage('fake-provider', 'fake-image')
  122. self.replace_config(configfile, 'node_two_provider.yaml')
  123. self.waitForImage('fake-provider2', 'fake-image')
  124. def test_provider_removal(self):
  125. configfile = self.setup_config('node_two_provider.yaml')
  126. self.useBuilder(configfile)
  127. self.waitForImage('fake-provider', 'fake-image')
  128. self.waitForImage('fake-provider2', 'fake-image')
  129. image = self.zk.getMostRecentImageUpload('fake-provider', 'fake-image')
  130. self.replace_config(configfile, 'node_two_provider_remove.yaml')
  131. self.waitForImageDeletion('fake-provider2', 'fake-image')
  132. image2 = self.zk.getMostRecentImageUpload('fake-provider',
  133. 'fake-image')
  134. self.assertEqual(image, image2)
  135. def test_image_addition(self):
  136. configfile = self.setup_config('node.yaml')
  137. self.useBuilder(configfile)
  138. self.waitForImage('fake-provider', 'fake-image')
  139. self.replace_config(configfile, 'node_two_image.yaml')
  140. self.waitForImage('fake-provider', 'fake-image2')
  141. def test_image_removal(self):
  142. configfile = self.setup_config('node_two_image.yaml')
  143. self.useBuilder(configfile)
  144. self.waitForImage('fake-provider', 'fake-image')
  145. self.waitForImage('fake-provider', 'fake-image2')
  146. self.replace_config(configfile, 'node_two_image_remove.yaml')
  147. self.waitForImageDeletion('fake-provider', 'fake-image2')
  148. self.waitForBuildDeletion('fake-image2', '0000000001')
  149. def test_image_rebuild_age(self):
  150. self._test_image_rebuild_age()
  151. def _test_image_rebuild_age(self, expire=86400):
  152. configfile = self.setup_config('node.yaml')
  153. self.useBuilder(configfile)
  154. build = self.waitForBuild('fake-image', '0000000001')
  155. log_path1 = os.path.join(self._config_build_log_dir.path,
  156. 'fake-image-0000000001.log')
  157. self.assertTrue(os.path.exists(log_path1))
  158. image = self.waitForImage('fake-provider', 'fake-image')
  159. # Expire rebuild-age (default: 1day) to force a new build.
  160. build.state_time -= expire
  161. with self.zk.imageBuildLock('fake-image', blocking=True, timeout=1):
  162. self.zk.storeBuild('fake-image', build, '0000000001')
  163. self.waitForBuild('fake-image', '0000000002')
  164. log_path2 = os.path.join(self._config_build_log_dir.path,
  165. 'fake-image-0000000002.log')
  166. self.assertTrue(os.path.exists(log_path2))
  167. self.waitForImage('fake-provider', 'fake-image', [image])
  168. builds = self.zk.getBuilds('fake-image', zk.READY)
  169. self.assertEqual(len(builds), 2)
  170. return (build, image)
  171. def test_image_rotation(self):
  172. # Expire rebuild-age (2days), to avoid problems when expiring 2 images.
  173. self._test_image_rebuild_age(expire=172800)
  174. build = self.waitForBuild('fake-image', '0000000002')
  175. # Expire rebuild-age (default: 1day) to force a new build.
  176. build.state_time -= 86400
  177. with self.zk.imageBuildLock('fake-image', blocking=True, timeout=1):
  178. self.zk.storeBuild('fake-image', build, '0000000002')
  179. self.waitForBuildDeletion('fake-image', '0000000001')
  180. self.waitForBuild('fake-image', '0000000003')
  181. log_path1 = os.path.join(self._config_build_log_dir.path,
  182. 'fake-image-0000000001.log')
  183. log_path2 = os.path.join(self._config_build_log_dir.path,
  184. 'fake-image-0000000002.log')
  185. log_path3 = os.path.join(self._config_build_log_dir.path,
  186. 'fake-image-0000000003.log')
  187. # Our log retention is set to 1, so the first log should be deleted.
  188. self.assertFalse(os.path.exists(log_path1))
  189. self.assertTrue(os.path.exists(log_path2))
  190. self.assertTrue(os.path.exists(log_path3))
  191. builds = self.zk.getBuilds('fake-image', zk.READY)
  192. self.assertEqual(len(builds), 2)
  193. def test_image_rotation_invalid_external_name(self):
  194. # NOTE(pabelanger): We are forcing fake-image to leak in fake-provider.
  195. # We do this to test our CleanupWorker will properly delete diskimage
  196. # builds from the HDD. For this test, we don't care about the leaked
  197. # image.
  198. #
  199. # Ensure we have a total of 3 diskimages on disk, so we can confirm
  200. # nodepool-builder will properly purge the 1 diskimage build leaving a
  201. # total of 2 diskimages on disk at all times.
  202. # Expire rebuild-age (2days), to avoid problems when expiring 2 images.
  203. build001, image001 = self._test_image_rebuild_age(expire=172800)
  204. build002 = self.waitForBuild('fake-image', '0000000002')
  205. # Make sure 2rd diskimage build was uploaded.
  206. image002 = self.waitForImage('fake-provider', 'fake-image', [image001])
  207. self.assertEqual(image002.build_id, '0000000002')
  208. # Delete external name / id so we can test exception handlers.
  209. upload = self.zk.getUploads(
  210. 'fake-image', '0000000001', 'fake-provider', zk.READY)[0]
  211. upload.external_name = None
  212. upload.external_id = None
  213. with self.zk.imageUploadLock(upload.image_name, upload.build_id,
  214. upload.provider_name, blocking=True,
  215. timeout=1):
  216. self.zk.storeImageUpload(upload.image_name, upload.build_id,
  217. upload.provider_name, upload, upload.id)
  218. # Expire rebuild-age (default: 1day) to force a new build.
  219. build002.state_time -= 86400
  220. with self.zk.imageBuildLock('fake-image', blocking=True, timeout=1):
  221. self.zk.storeBuild('fake-image', build002, '0000000002')
  222. self.waitForBuildDeletion('fake-image', '0000000001')
  223. # Make sure fake-image for fake-provider is removed from zookeeper.
  224. upload = self.zk.getUploads(
  225. 'fake-image', '0000000001', 'fake-provider')
  226. self.assertEqual(len(upload), 0)
  227. self.waitForBuild('fake-image', '0000000003')
  228. # Ensure we only have 2 builds on disk.
  229. builds = self.zk.getBuilds('fake-image', zk.READY)
  230. self.assertEqual(len(builds), 2)
  231. # Make sure 3rd diskimage build was uploaded.
  232. image003 = self.waitForImage(
  233. 'fake-provider', 'fake-image', [image001, image002])
  234. self.assertEqual(image003.build_id, '0000000003')
  235. def test_cleanup_hard_upload_fails(self):
  236. configfile = self.setup_config('node.yaml')
  237. self.useBuilder(configfile)
  238. self.waitForImage('fake-provider', 'fake-image')
  239. upload = self.zk.getUploads('fake-image', '0000000001',
  240. 'fake-provider', zk.READY)[0]
  241. # Store a new ZK node as UPLOADING to represent a hard fail
  242. upload.state = zk.UPLOADING
  243. with self.zk.imageUploadLock(upload.image_name, upload.build_id,
  244. upload.provider_name, blocking=True,
  245. timeout=1):
  246. upnum = self.zk.storeImageUpload(upload.image_name,
  247. upload.build_id,
  248. upload.provider_name,
  249. upload)
  250. # Now it should disappear from the current build set of uploads
  251. self.waitForUploadRecordDeletion(upload.provider_name,
  252. upload.image_name,
  253. upload.build_id,
  254. upnum)
  255. def test_cleanup_failed_image_build(self):
  256. configfile = self.setup_config('node_diskimage_fail.yaml')
  257. self.useBuilder(configfile)
  258. # NOTE(pabelanger): We are racing here, but don't really care. We just
  259. # need our first image build to fail.
  260. self.replace_config(configfile, 'node.yaml')
  261. self.waitForImage('fake-provider', 'fake-image')
  262. # Make sure our cleanup worker properly removes the first build.
  263. self.waitForBuildDeletion('fake-image', '0000000001')
  264. self.assertReportedStat('nodepool.dib_image_build.fake-image.qcow2.rc',
  265. '127', 'g')
  266. self.assertReportedStat('nodepool.dib_image_build.'
  267. 'fake-image.qcow2.duration', None, 'ms')
  268. def test_diskimage_build_only(self):
  269. configfile = self.setup_config('node_diskimage_only.yaml')
  270. self.useBuilder(configfile)
  271. build_tar = self.waitForBuild('fake-image', '0000000001')
  272. build_default = self.waitForBuild('fake-image-default-format',
  273. '0000000001')
  274. self.assertEqual(build_tar._formats, ['tar'])
  275. self.assertEqual(build_default._formats, ['qcow2'])
  276. self.assertReportedStat('nodepool.dib_image_build.fake-image.tar.rc',
  277. '0', 'g')
  278. self.assertReportedStat('nodepool.dib_image_build.'
  279. 'fake-image.tar.duration', None, 'ms')
  280. def test_diskimage_build_formats(self):
  281. configfile = self.setup_config('node_diskimage_formats.yaml')
  282. self.useBuilder(configfile)
  283. build_default = self.waitForBuild('fake-image-default-format',
  284. '0000000001')
  285. build_vhd = self.waitForBuild('fake-image-vhd', '0000000001')
  286. self.assertEqual(build_default._formats, ['qcow2'])
  287. self.assertEqual(build_vhd._formats, ['vhd'])
  288. @mock.patch.object(subprocess.Popen, 'wait')
  289. def test_diskimage_build_timeout(self, mock_wait):
  290. mock_wait.side_effect = subprocess.TimeoutExpired('dib_cmd', 1)
  291. configfile = self.setup_config('diskimage_build_timeout.yaml')
  292. self.useBuilder(configfile, cleanup_interval=0)
  293. self.waitForBuild('fake-image', '0000000001', states=(zk.FAILED,))