Kolla provides production-ready containers and deployment tools for operating OpenStack clouds
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

test_build.py 16KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421
  1. # Licensed under the Apache License, Version 2.0 (the "License");
  2. # you may not use this file except in compliance with the License.
  3. # You may obtain a copy of the License at
  4. #
  5. # http://www.apache.org/licenses/LICENSE-2.0
  6. #
  7. # Unless required by applicable law or agreed to in writing, software
  8. # distributed under the License is distributed on an "AS IS" BASIS,
  9. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. # See the License for the specific language governing permissions and
  11. # limitations under the License.
  12. import fixtures
  13. import itertools
  14. import mock
  15. import os
  16. import requests
  17. import sys
  18. from kolla.cmd import build as build_cmd
  19. from kolla import exception
  20. from kolla.image import build
  21. from kolla.tests import base
  22. FAKE_IMAGE = build.Image(
  23. 'image-base', 'image-base:latest',
  24. '/fake/path', parent_name=None,
  25. parent=None, status=build.STATUS_MATCHED)
  26. FAKE_IMAGE_CHILD = build.Image(
  27. 'image-child', 'image-child:latest',
  28. '/fake/path2', parent_name='image-base',
  29. parent=FAKE_IMAGE, status=build.STATUS_MATCHED)
  30. FAKE_IMAGE_CHILD_UNMATCHED = build.Image(
  31. 'image-child-unmatched', 'image-child-unmatched:latest',
  32. '/fake/path3', parent_name='image-base',
  33. parent=FAKE_IMAGE, status=build.STATUS_UNMATCHED)
  34. FAKE_IMAGE_CHILD_ERROR = build.Image(
  35. 'image-child-error', 'image-child-error:latest',
  36. '/fake/path4', parent_name='image-base',
  37. parent=FAKE_IMAGE, status=build.STATUS_ERROR)
  38. FAKE_IMAGE_CHILD_BUILT = build.Image(
  39. 'image-child-built', 'image-child-built:latest',
  40. '/fake/path5', parent_name='image-base',
  41. parent=FAKE_IMAGE, status=build.STATUS_BUILT)
  42. class TasksTest(base.TestCase):
  43. def setUp(self):
  44. super(TasksTest, self).setUp()
  45. self.image = FAKE_IMAGE.copy()
  46. # NOTE(jeffrey4l): use a real, temporary dir
  47. self.image.path = self.useFixture(fixtures.TempDir()).path
  48. self.imageChild = FAKE_IMAGE_CHILD.copy()
  49. # NOTE(mandre) we want the local copy of FAKE_IMAGE as the parent
  50. self.imageChild.parent = self.image
  51. self.imageChild.path = self.useFixture(fixtures.TempDir()).path
  52. @mock.patch.dict(os.environ, clear=True)
  53. @mock.patch('docker.Client')
  54. def test_push_image(self, mock_client):
  55. self.dc = mock_client
  56. pusher = build.PushTask(self.conf, self.image)
  57. pusher.run()
  58. mock_client().push.assert_called_once_with(
  59. self.image.canonical_name, stream=True, insecure_registry=True)
  60. @mock.patch.dict(os.environ, clear=True)
  61. @mock.patch('docker.Client')
  62. def test_build_image(self, mock_client):
  63. self.dc = mock_client
  64. push_queue = mock.Mock()
  65. builder = build.BuildTask(self.conf, self.image, push_queue)
  66. builder.run()
  67. mock_client().build.assert_called_once_with(
  68. path=self.image.path, tag=self.image.canonical_name,
  69. nocache=False, rm=True, pull=True, forcerm=True,
  70. buildargs=None)
  71. self.assertTrue(builder.success)
  72. @mock.patch.dict(os.environ, clear=True)
  73. @mock.patch('docker.Client')
  74. def test_build_image_with_build_arg(self, mock_client):
  75. self.dc = mock_client
  76. build_args = {
  77. 'HTTP_PROXY': 'http://localhost:8080',
  78. 'NO_PROXY': '127.0.0.1'
  79. }
  80. self.conf.set_override('build_args', build_args)
  81. push_queue = mock.Mock()
  82. builder = build.BuildTask(self.conf, self.image, push_queue)
  83. builder.run()
  84. mock_client().build.assert_called_once_with(
  85. path=self.image.path, tag=self.image.canonical_name,
  86. nocache=False, rm=True, pull=True, forcerm=True,
  87. buildargs=build_args)
  88. self.assertTrue(builder.success)
  89. @mock.patch.dict(os.environ, {'http_proxy': 'http://FROM_ENV:8080'},
  90. clear=True)
  91. @mock.patch('docker.Client')
  92. def test_build_arg_from_env(self, mock_client):
  93. push_queue = mock.Mock()
  94. self.dc = mock_client
  95. build_args = {
  96. 'http_proxy': 'http://FROM_ENV:8080',
  97. }
  98. builder = build.BuildTask(self.conf, self.image, push_queue)
  99. builder.run()
  100. mock_client().build.assert_called_once_with(
  101. path=self.image.path, tag=self.image.canonical_name,
  102. nocache=False, rm=True, pull=True, forcerm=True,
  103. buildargs=build_args)
  104. self.assertTrue(builder.success)
  105. @mock.patch.dict(os.environ, {'http_proxy': 'http://FROM_ENV:8080'},
  106. clear=True)
  107. @mock.patch('docker.Client')
  108. def test_build_arg_precedence(self, mock_client):
  109. self.dc = mock_client
  110. build_args = {
  111. 'http_proxy': 'http://localhost:8080',
  112. }
  113. self.conf.set_override('build_args', build_args)
  114. push_queue = mock.Mock()
  115. builder = build.BuildTask(self.conf, self.image, push_queue)
  116. builder.run()
  117. mock_client().build.assert_called_once_with(
  118. path=self.image.path, tag=self.image.canonical_name,
  119. nocache=False, rm=True, pull=True, forcerm=True,
  120. buildargs=build_args)
  121. self.assertTrue(builder.success)
  122. @mock.patch('docker.Client')
  123. @mock.patch('requests.get')
  124. def test_requests_get_timeout(self, mock_get, mock_client):
  125. self.dc = mock_client
  126. self.image.source = {
  127. 'source': 'http://fake/source',
  128. 'type': 'url',
  129. 'name': 'fake-image-base'
  130. }
  131. push_queue = mock.Mock()
  132. builder = build.BuildTask(self.conf, self.image, push_queue)
  133. mock_get.side_effect = requests.exceptions.Timeout
  134. get_result = builder.process_source(self.image, self.image.source)
  135. self.assertIsNone(get_result)
  136. self.assertEqual(self.image.status, build.STATUS_ERROR)
  137. mock_get.assert_called_once_with(self.image.source['source'],
  138. timeout=120)
  139. self.assertFalse(builder.success)
  140. @mock.patch('docker.Client')
  141. @mock.patch('requests.get')
  142. @mock.patch('shutil.rmtree')
  143. @mock.patch('shutil.copyfile')
  144. @mock.patch('os.utime')
  145. def test_process_source(self, mock_get, mock_client,
  146. mock_rmtree, mock_copyfile, mock_utime):
  147. for source in [{'source': 'http://fake/source1', 'type': 'url',
  148. 'name': 'fake-image-base1',
  149. 'reference': 'http://fake/reference1'},
  150. {'source': 'http://fake/source2', 'type': 'git',
  151. 'name': 'fake-image-base2',
  152. 'reference': 'http://fake/reference2'},
  153. {'source': 'http://fake/source3', 'type': 'local',
  154. 'name': 'fake-image-base3',
  155. 'reference': 'http://fake/reference3'},
  156. {'source': 'http://fake/source4', 'type': None,
  157. 'name': 'fake-image-base4',
  158. 'reference': 'http://fake/reference4'}]:
  159. self.image.source = source
  160. push_queue = mock.Mock()
  161. builder = build.BuildTask(self.conf, self.image, push_queue)
  162. get_result = builder.process_source(self.image, self.image.source)
  163. self.assertEqual(self.image.status, build.STATUS_ERROR)
  164. self.assertFalse(builder.success)
  165. if source['type'] != 'local':
  166. self.assertIsNone(get_result)
  167. else:
  168. self.assertIsNotNone(get_result)
  169. @mock.patch('docker.Client')
  170. def test_followups_docker_image(self, mock_client):
  171. self.imageChild.source = {
  172. 'source': 'http://fake/source',
  173. 'type': 'url',
  174. 'name': 'fake-image-base'
  175. }
  176. self.imageChild.children.append(FAKE_IMAGE_CHILD_UNMATCHED)
  177. push_queue = mock.Mock()
  178. builder = build.BuildTask(self.conf, self.imageChild, push_queue)
  179. builder.success = True
  180. self.conf.push = FAKE_IMAGE_CHILD_UNMATCHED
  181. get_result = builder.followups
  182. self.assertEqual(1, len(get_result))
  183. class KollaWorkerTest(base.TestCase):
  184. config_file = 'default.conf'
  185. def setUp(self):
  186. super(KollaWorkerTest, self).setUp()
  187. image = FAKE_IMAGE.copy()
  188. image.status = None
  189. image_child = FAKE_IMAGE_CHILD.copy()
  190. image_child.status = None
  191. image_unmatched = FAKE_IMAGE_CHILD_UNMATCHED.copy()
  192. image_error = FAKE_IMAGE_CHILD_ERROR.copy()
  193. image_built = FAKE_IMAGE_CHILD_BUILT.copy()
  194. # NOTE(mandre) we want the local copy of FAKE_IMAGE as the parent
  195. for i in [image_child, image_unmatched, image_error, image_built]:
  196. i.parent = image
  197. self.images = [image, image_child, image_unmatched,
  198. image_error, image_built]
  199. def test_supported_base_type(self):
  200. rh_base = ['centos', 'oraclelinux', 'rhel']
  201. rh_type = ['source', 'binary', 'rdo', 'rhos']
  202. deb_base = ['ubuntu', 'debian']
  203. deb_type = ['source', 'binary']
  204. for base_distro, install_type in itertools.chain(
  205. itertools.product(rh_base, rh_type),
  206. itertools.product(deb_base, deb_type)):
  207. self.conf.set_override('base', base_distro)
  208. self.conf.set_override('install_type', install_type)
  209. # should no exception raised
  210. build.KollaWorker(self.conf)
  211. def test_unsupported_base_type(self):
  212. for base_distro, install_type in itertools.product(
  213. ['ubuntu', 'debian'], ['rdo', 'rhos']):
  214. self.conf.set_override('base', base_distro)
  215. self.conf.set_override('install_type', install_type)
  216. self.assertRaises(exception.KollaMismatchBaseTypeException,
  217. build.KollaWorker, self.conf)
  218. def test_build_image_list_adds_plugins(self):
  219. self.conf.set_override('install_type', 'source')
  220. kolla = build.KollaWorker(self.conf)
  221. kolla.setup_working_dir()
  222. kolla.find_dockerfiles()
  223. kolla.create_dockerfiles()
  224. kolla.build_image_list()
  225. expected_plugin = {
  226. 'name': 'neutron-server-plugin-networking-arista',
  227. 'reference': 'master',
  228. 'source': 'https://git.openstack.org/openstack/networking-arista',
  229. 'type': 'git'
  230. }
  231. found = False
  232. for image in kolla.images:
  233. if image.name == 'neutron-server':
  234. for plugin in image.plugins:
  235. if plugin == expected_plugin:
  236. found = True
  237. break
  238. break
  239. if not found:
  240. self.fail('Can not find the expected neutron arista plugin')
  241. def test_build_image_list_plugin_parsing(self):
  242. """Ensure regex used to parse plugins adds them to the correct image"""
  243. self.conf.set_override('install_type', 'source')
  244. kolla = build.KollaWorker(self.conf)
  245. kolla.setup_working_dir()
  246. kolla.find_dockerfiles()
  247. kolla.create_dockerfiles()
  248. kolla.build_image_list()
  249. for image in kolla.images:
  250. if image.name == 'base':
  251. self.assertEqual(len(image.plugins), 0,
  252. 'base image should not have any plugins '
  253. 'registered')
  254. break
  255. else:
  256. self.fail('Expected to find the base image in this test')
  257. def test_set_time(self):
  258. self.conf.set_override('install_type', 'source')
  259. kolla = build.KollaWorker(self.conf)
  260. kolla.setup_working_dir()
  261. kolla.set_time()
  262. def _get_matched_images(self, images):
  263. return [image for image in images
  264. if image.status == build.STATUS_MATCHED]
  265. def test_skip_parents(self):
  266. self.conf.set_override('regex', ['image-child'])
  267. self.conf.set_override('skip_parents', True)
  268. kolla = build.KollaWorker(self.conf)
  269. kolla.images = self.images
  270. kolla.filter_images()
  271. self.assertEqual(build.STATUS_SKIPPED, kolla.images[1].parent.status)
  272. def test_without_profile(self):
  273. kolla = build.KollaWorker(self.conf)
  274. kolla.images = self.images
  275. kolla.filter_images()
  276. self.assertEqual(5, len(self._get_matched_images(kolla.images)))
  277. def test_build_rpm_setup(self):
  278. """checking the length of list of docker commands"""
  279. self.conf.set_override('rpm_setup_config', ["a.rpm", "b.repo"])
  280. kolla = build.KollaWorker(self.conf)
  281. self.assertEqual(2, len(kolla.rpm_setup))
  282. def test_pre_defined_exist_profile(self):
  283. # default profile include the fake image: image-base
  284. self.conf.set_override('profile', ['default'])
  285. kolla = build.KollaWorker(self.conf)
  286. kolla.images = self.images
  287. kolla.filter_images()
  288. self.assertEqual(1, len(self._get_matched_images(kolla.images)))
  289. def test_pre_defined_exist_profile_not_include(self):
  290. # infra profile do not include the fake image: image-base
  291. self.conf.set_override('profile', ['infra'])
  292. kolla = build.KollaWorker(self.conf)
  293. kolla.images = self.images
  294. kolla.filter_images()
  295. self.assertEqual(0, len(self._get_matched_images(kolla.images)))
  296. def test_pre_defined_not_exist_profile(self):
  297. # NOTE(jeffrey4l): not exist profile will raise ValueError
  298. self.conf.set_override('profile', ['not_exist'])
  299. kolla = build.KollaWorker(self.conf)
  300. kolla.images = self.images
  301. self.assertRaises(ValueError,
  302. kolla.filter_images)
  303. @mock.patch('json.dump')
  304. def test_list_dependencies(self, dump_mock):
  305. self.conf.set_override('profile', ['all'])
  306. kolla = build.KollaWorker(self.conf)
  307. kolla.images = self.images
  308. kolla.filter_images()
  309. kolla.list_dependencies()
  310. dump_mock.assert_called_once_with(mock.ANY, sys.stdout, indent=2)
  311. def test_summary(self):
  312. kolla = build.KollaWorker(self.conf)
  313. kolla.images = self.images
  314. kolla.image_statuses_good['good'] = None
  315. kolla.image_statuses_bad['bad'] = None
  316. kolla.image_statuses_unmatched['unmatched'] = None
  317. results = kolla.summary()
  318. self.assertIsNone(results['failed'][0]['status'])
  319. @mock.patch('shutil.copytree')
  320. def test_work_dir(self, copytree_mock):
  321. self.conf.set_override('work_dir', '/tmp/foo')
  322. kolla = build.KollaWorker(self.conf)
  323. kolla.setup_working_dir()
  324. self.assertEqual('/tmp/foo/docker', kolla.working_dir)
  325. class MainTest(base.TestCase):
  326. @mock.patch.object(build, 'run_build')
  327. def test_images_built(self, mock_run_build):
  328. image_statuses = ({}, {'img': 'built'}, {}, {})
  329. mock_run_build.return_value = image_statuses
  330. result = build_cmd.main()
  331. self.assertEqual(0, result)
  332. @mock.patch.object(build, 'run_build')
  333. def test_images_unmatched(self, mock_run_build):
  334. image_statuses = ({}, {}, {'img': 'unmatched'}, {})
  335. mock_run_build.return_value = image_statuses
  336. result = build_cmd.main()
  337. self.assertEqual(0, result)
  338. @mock.patch.object(build, 'run_build')
  339. def test_no_images_built(self, mock_run_build):
  340. mock_run_build.return_value = None
  341. result = build_cmd.main()
  342. self.assertEqual(0, result)
  343. @mock.patch.object(build, 'run_build')
  344. def test_bad_images(self, mock_run_build):
  345. image_statuses = ({'img': 'error'}, {}, {}, {})
  346. mock_run_build.return_value = image_statuses
  347. result = build_cmd.main()
  348. self.assertEqual(1, result)
  349. @mock.patch('sys.argv')
  350. def test_run_build(self, mock_sys):
  351. result = build.run_build()
  352. self.assertTrue(result)
  353. @mock.patch.object(build, 'run_build')
  354. def test_skipped_images(self, mock_run_build):
  355. image_statuses = ({}, {}, {}, {'img': 'skipped'})
  356. mock_run_build.return_value = image_statuses
  357. result = build_cmd.main()
  358. self.assertEqual(0, result)