The Gatekeeper, or a project gating system
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

960 lines
37 KiB

  1. # Copyright 2012 Hewlett-Packard Development Company, L.P.
  2. # Copyright 2018 Red Hat, Inc.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. from tests.base import (
  16. ZuulTestCase,
  17. )
  18. class TestGerritToGithubCRD(ZuulTestCase):
  19. config_file = 'zuul-gerrit-github.conf'
  20. tenant_config_file = 'config/cross-source/main.yaml'
  21. def test_crd_gate(self):
  22. "Test cross-repo dependencies"
  23. A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
  24. B = self.fake_github.openFakePullRequest('github/project2', 'master',
  25. 'B')
  26. A.addApproval('Code-Review', 2)
  27. AM2 = self.fake_gerrit.addFakeChange('gerrit/project1', 'master',
  28. 'AM2')
  29. AM1 = self.fake_gerrit.addFakeChange('gerrit/project1', 'master',
  30. 'AM1')
  31. AM2.setMerged()
  32. AM1.setMerged()
  33. # A -> AM1 -> AM2
  34. # A Depends-On: B
  35. # M2 is here to make sure it is never queried. If it is, it
  36. # means zuul is walking down the entire history of merged
  37. # changes.
  38. A.setDependsOn(AM1, 1)
  39. AM1.setDependsOn(AM2, 1)
  40. A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  41. A.subject, B.url)
  42. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  43. self.waitUntilSettled()
  44. self.assertEqual(A.data['status'], 'NEW')
  45. self.assertFalse(B.is_merged)
  46. for connection in self.scheds.first.connections.connections.values():
  47. connection.maintainCache([])
  48. self.executor_server.hold_jobs_in_build = True
  49. B.addLabel('approved')
  50. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  51. self.waitUntilSettled()
  52. self.executor_server.release('.*-merge')
  53. self.waitUntilSettled()
  54. self.executor_server.release('.*-merge')
  55. self.waitUntilSettled()
  56. self.executor_server.hold_jobs_in_build = False
  57. self.executor_server.release()
  58. self.waitUntilSettled()
  59. self.assertEqual(AM2.queried, 0)
  60. self.assertEqual(A.data['status'], 'MERGED')
  61. self.assertTrue(B.is_merged)
  62. self.assertEqual(A.reported, 2)
  63. self.assertEqual(len(B.comments), 2)
  64. changes = self.getJobFromHistory(
  65. 'project-merge', 'gerrit/project1').changes
  66. self.assertEqual(changes, '1,%s 1,1' % B.head_sha)
  67. def test_crd_branch(self):
  68. "Test cross-repo dependencies in multiple branches"
  69. self.create_branch('github/project2', 'mp')
  70. A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
  71. B = self.fake_github.openFakePullRequest('github/project2', 'master',
  72. 'B')
  73. C1 = self.fake_github.openFakePullRequest('github/project2', 'mp',
  74. 'C1')
  75. A.addApproval('Code-Review', 2)
  76. # A Depends-On: B+C1
  77. A.data['commitMessage'] = '%s\n\nDepends-On: %s\nDepends-On: %s\n' % (
  78. A.subject, B.url, C1.url)
  79. self.executor_server.hold_jobs_in_build = True
  80. B.addLabel('approved')
  81. C1.addLabel('approved')
  82. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  83. self.waitUntilSettled()
  84. self.executor_server.release('.*-merge')
  85. self.waitUntilSettled()
  86. self.executor_server.release('.*-merge')
  87. self.waitUntilSettled()
  88. self.executor_server.release('.*-merge')
  89. self.waitUntilSettled()
  90. self.executor_server.hold_jobs_in_build = False
  91. self.executor_server.release()
  92. self.waitUntilSettled()
  93. self.assertEqual(A.data['status'], 'MERGED')
  94. self.assertTrue(B.is_merged)
  95. self.assertTrue(C1.is_merged)
  96. self.assertEqual(A.reported, 2)
  97. self.assertEqual(len(B.comments), 2)
  98. self.assertEqual(len(C1.comments), 2)
  99. changes = self.getJobFromHistory(
  100. 'project-merge', 'gerrit/project1').changes
  101. self.assertEqual(changes, '1,%s 2,%s 1,1' %
  102. (B.head_sha, C1.head_sha))
  103. def test_crd_gate_reverse(self):
  104. "Test reverse cross-repo dependencies"
  105. A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
  106. B = self.fake_github.openFakePullRequest('github/project2', 'master',
  107. 'B')
  108. A.addApproval('Code-Review', 2)
  109. # A Depends-On: B
  110. A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  111. A.subject, B.url)
  112. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  113. self.waitUntilSettled()
  114. self.assertEqual(A.data['status'], 'NEW')
  115. self.assertFalse(B.is_merged)
  116. self.executor_server.hold_jobs_in_build = True
  117. A.addApproval('Approved', 1)
  118. self.fake_github.emitEvent(B.addLabel('approved'))
  119. self.waitUntilSettled()
  120. self.executor_server.release('.*-merge')
  121. self.waitUntilSettled()
  122. self.executor_server.release('.*-merge')
  123. self.waitUntilSettled()
  124. self.executor_server.hold_jobs_in_build = False
  125. self.executor_server.release()
  126. self.waitUntilSettled()
  127. self.assertEqual(A.data['status'], 'MERGED')
  128. self.assertTrue(B.is_merged)
  129. self.assertEqual(A.reported, 2)
  130. self.assertEqual(len(B.comments), 2)
  131. changes = self.getJobFromHistory(
  132. 'project-merge', 'gerrit/project1').changes
  133. self.assertEqual(changes, '1,%s 1,1' %
  134. (B.head_sha,))
  135. def test_crd_cycle(self):
  136. "Test cross-repo dependency cycles"
  137. A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
  138. msg = "Depends-On: %s" % (A.data['url'],)
  139. B = self.fake_github.openFakePullRequest('github/project2', 'master',
  140. 'B', body=msg)
  141. A.addApproval('Code-Review', 2)
  142. B.addLabel('approved')
  143. # A -> B -> A (via commit-depends)
  144. A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  145. A.subject, B.url)
  146. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  147. self.waitUntilSettled()
  148. self.assertEqual(A.reported, 1)
  149. self.assertEqual(
  150. A.messages[0],
  151. "Build failed.\n\n\nWarning:\n Dependency cycle detected\n")
  152. self.assertEqual(len(B.comments), 0)
  153. self.assertEqual(A.data['status'], 'NEW')
  154. self.assertFalse(B.is_merged)
  155. def test_crd_gate_unknown(self):
  156. "Test unknown projects in dependent pipeline"
  157. self.init_repo("github/unknown", tag='init')
  158. A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
  159. B = self.fake_github.openFakePullRequest('github/unknown', 'master',
  160. 'B')
  161. A.addApproval('Code-Review', 2)
  162. # A Depends-On: B
  163. A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  164. A.subject, B.url)
  165. event = B.addLabel('approved')
  166. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  167. self.waitUntilSettled()
  168. # Unknown projects cannot share a queue with any other
  169. # since they don't have common jobs with any other (they have no jobs).
  170. # Changes which depend on unknown project changes
  171. # should not be processed in dependent pipeline
  172. self.assertEqual(A.data['status'], 'NEW')
  173. self.assertFalse(B.is_merged)
  174. self.assertEqual(A.reported, 0)
  175. self.assertEqual(len(B.comments), 0)
  176. self.assertEqual(len(self.history), 0)
  177. # Simulate change B being gated outside this layout Set the
  178. # change merged before submitting the event so that when the
  179. # event triggers a gerrit query to update the change, we get
  180. # the information that it was merged.
  181. B.setMerged('merged')
  182. self.fake_github.emitEvent(event)
  183. self.waitUntilSettled()
  184. self.assertEqual(len(self.history), 0)
  185. # Now that B is merged, A should be able to be enqueued and
  186. # merged.
  187. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  188. self.waitUntilSettled()
  189. self.assertEqual(A.data['status'], 'MERGED')
  190. self.assertEqual(A.reported, 2)
  191. self.assertTrue(B.is_merged)
  192. self.assertEqual(len(B.comments), 0)
  193. def test_crd_check(self):
  194. "Test cross-repo dependencies in independent pipelines"
  195. self.executor_server.hold_jobs_in_build = True
  196. self.gearman_server.hold_jobs_in_queue = True
  197. A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
  198. B = self.fake_github.openFakePullRequest(
  199. 'github/project2', 'master', 'B')
  200. # A Depends-On: B
  201. A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  202. A.subject, B.url)
  203. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  204. self.waitUntilSettled()
  205. self.gearman_server.hold_jobs_in_queue = False
  206. self.gearman_server.release()
  207. self.waitUntilSettled()
  208. self.executor_server.release('.*-merge')
  209. self.waitUntilSettled()
  210. self.assertTrue(self.builds[0].hasChanges(A, B))
  211. self.executor_server.hold_jobs_in_build = False
  212. self.executor_server.release()
  213. self.waitUntilSettled()
  214. self.assertEqual(A.data['status'], 'NEW')
  215. self.assertFalse(B.is_merged)
  216. self.assertEqual(A.reported, 1)
  217. self.assertEqual(len(B.comments), 0)
  218. changes = self.getJobFromHistory(
  219. 'project-merge', 'gerrit/project1').changes
  220. self.assertEqual(changes, '1,%s 1,1' %
  221. (B.head_sha,))
  222. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  223. self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
  224. def test_crd_check_duplicate(self):
  225. "Test duplicate check in independent pipelines"
  226. self.executor_server.hold_jobs_in_build = True
  227. A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
  228. B = self.fake_github.openFakePullRequest(
  229. 'github/project2', 'master', 'B')
  230. self.waitUntilSettled()
  231. # A Depends-On: B
  232. A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  233. A.subject, B.url)
  234. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  235. check_pipeline = tenant.layout.pipelines['check']
  236. # Add two dependent changes...
  237. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  238. self.waitUntilSettled()
  239. self.assertEqual(len(check_pipeline.getAllItems()), 2)
  240. # ...make sure the live one is not duplicated...
  241. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  242. self.waitUntilSettled()
  243. self.assertEqual(len(check_pipeline.getAllItems()), 2)
  244. # ...but the non-live one is able to be.
  245. self.fake_github.emitEvent(B.getPullRequestEditedEvent())
  246. self.waitUntilSettled()
  247. self.assertEqual(len(check_pipeline.getAllItems()), 3)
  248. # Release jobs in order to avoid races with change A jobs
  249. # finishing before change B jobs.
  250. self.orderedRelease()
  251. self.executor_server.hold_jobs_in_build = False
  252. self.executor_server.release()
  253. self.waitUntilSettled()
  254. self.assertEqual(A.data['status'], 'NEW')
  255. self.assertFalse(B.is_merged)
  256. self.assertEqual(A.reported, 1)
  257. self.assertEqual(len(B.comments), 1)
  258. changes = self.getJobFromHistory(
  259. 'project-merge', 'gerrit/project1').changes
  260. self.assertEqual(changes, '1,%s 1,1' %
  261. (B.head_sha,))
  262. changes = self.getJobFromHistory(
  263. 'project-merge', 'github/project2').changes
  264. self.assertEqual(changes, '1,%s' %
  265. (B.head_sha,))
  266. self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
  267. self.assertIn('Build succeeded', A.messages[0])
  268. def _test_crd_check_reconfiguration(self, project1, project2):
  269. "Test cross-repo dependencies re-enqueued in independent pipelines"
  270. self.gearman_server.hold_jobs_in_queue = True
  271. A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
  272. B = self.fake_github.openFakePullRequest(
  273. 'github/project2', 'master', 'B')
  274. # A Depends-On: B
  275. A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  276. A.subject, B.url)
  277. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  278. self.waitUntilSettled()
  279. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  280. # Make sure the items still share a change queue, and the
  281. # first one is not live.
  282. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  283. self.assertEqual(len(tenant.layout.pipelines['check'].queues), 1)
  284. queue = tenant.layout.pipelines['check'].queues[0]
  285. first_item = queue.queue[0]
  286. for item in queue.queue:
  287. self.assertEqual(item.queue, first_item.queue)
  288. self.assertFalse(first_item.live)
  289. self.assertTrue(queue.queue[1].live)
  290. self.gearman_server.hold_jobs_in_queue = False
  291. self.gearman_server.release()
  292. self.waitUntilSettled()
  293. self.assertEqual(A.data['status'], 'NEW')
  294. self.assertFalse(B.is_merged)
  295. self.assertEqual(A.reported, 1)
  296. self.assertEqual(len(B.comments), 0)
  297. changes = self.getJobFromHistory(
  298. 'project-merge', 'gerrit/project1').changes
  299. self.assertEqual(changes, '1,%s 1,1' %
  300. (B.head_sha,))
  301. self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
  302. def test_crd_check_reconfiguration(self):
  303. self._test_crd_check_reconfiguration('org/project1', 'org/project2')
  304. def test_crd_undefined_project(self):
  305. """Test that undefined projects in dependencies are handled for
  306. independent pipelines"""
  307. # It's a hack for fake github,
  308. # as it implies repo creation upon the creation of any change
  309. self.init_repo("github/unknown", tag='init')
  310. self._test_crd_check_reconfiguration('gerrit/project1',
  311. 'github/unknown')
  312. def test_crd_check_transitive(self):
  313. "Test transitive cross-repo dependencies"
  314. # Specifically, if A -> B -> C, and C gets a new patchset and
  315. # A gets a new patchset, ensure the test of A,2 includes B,1
  316. # and C,2 (not C,1 which would indicate stale data in the
  317. # cache for B).
  318. A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
  319. C = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'C')
  320. # B Depends-On: C
  321. msg = "Depends-On: %s" % (C.data['url'],)
  322. B = self.fake_github.openFakePullRequest(
  323. 'github/project2', 'master', 'B', body=msg)
  324. # A Depends-On: B
  325. A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  326. A.subject, B.url)
  327. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  328. self.waitUntilSettled()
  329. self.assertEqual(self.history[-1].changes, '2,1 1,%s 1,1' %
  330. (B.head_sha,))
  331. self.fake_github.emitEvent(B.getPullRequestEditedEvent())
  332. self.waitUntilSettled()
  333. self.assertEqual(self.history[-1].changes, '2,1 1,%s' %
  334. (B.head_sha,))
  335. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  336. self.waitUntilSettled()
  337. self.assertEqual(self.history[-1].changes, '2,1')
  338. C.addPatchset()
  339. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(2))
  340. self.waitUntilSettled()
  341. self.assertEqual(self.history[-1].changes, '2,2')
  342. A.addPatchset()
  343. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
  344. self.waitUntilSettled()
  345. self.assertEqual(self.history[-1].changes, '2,2 1,%s 1,2' %
  346. (B.head_sha,))
  347. def test_crd_check_unknown(self):
  348. "Test unknown projects in independent pipeline"
  349. self.init_repo("github/unknown", tag='init')
  350. A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
  351. B = self.fake_github.openFakePullRequest(
  352. 'github/unknown', 'master', 'B')
  353. # A Depends-On: B
  354. A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  355. A.subject, B.url)
  356. # Make sure zuul has seen an event on B. This is necessary
  357. # in order to populate our fake github project db.
  358. self.fake_github.emitEvent(B.getPullRequestEditedEvent())
  359. # Note we wait until settled here as the event processing for
  360. # the next event may not have the updated db yet otherwise.
  361. self.waitUntilSettled()
  362. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  363. self.waitUntilSettled()
  364. self.assertEqual(A.data['status'], 'NEW')
  365. self.assertEqual(A.reported, 1)
  366. self.assertFalse(B.is_merged)
  367. self.assertEqual(len(B.comments), 0)
  368. def test_crd_cycle_join(self):
  369. "Test an updated change creates a cycle"
  370. A = self.fake_github.openFakePullRequest(
  371. 'github/project2', 'master', 'A')
  372. self.fake_github.emitEvent(A.getPullRequestEditedEvent())
  373. self.waitUntilSettled()
  374. self.assertEqual(len(A.comments), 1)
  375. # Create B->A
  376. B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B')
  377. B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  378. B.subject, A.url)
  379. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  380. self.waitUntilSettled()
  381. # Dep is there so zuul should have reported on B
  382. self.assertEqual(B.reported, 1)
  383. # Update A to add A->B (a cycle).
  384. A.editBody('Depends-On: %s\n' % (B.data['url']))
  385. self.fake_github.emitEvent(A.getPullRequestEditedEvent())
  386. self.waitUntilSettled()
  387. # Dependency cycle injected so zuul should have reported again on A
  388. self.assertEqual(len(A.comments), 2)
  389. # Now if we update B to remove the depends-on, everything
  390. # should be okay. B; A->B
  391. B.addPatchset()
  392. B.data['commitMessage'] = '%s\n' % (B.subject,)
  393. self.fake_github.emitEvent(A.getPullRequestEditedEvent())
  394. self.waitUntilSettled()
  395. # Cycle was removed so now zuul should have reported again on A
  396. self.assertEqual(len(A.comments), 3)
  397. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
  398. self.waitUntilSettled()
  399. self.assertEqual(B.reported, 2)
  400. class TestGithubToGerritCRD(ZuulTestCase):
  401. config_file = 'zuul-gerrit-github.conf'
  402. tenant_config_file = 'config/cross-source/main.yaml'
  403. def test_crd_gate(self):
  404. "Test cross-repo dependencies"
  405. A = self.fake_github.openFakePullRequest('github/project2', 'master',
  406. 'A')
  407. B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B')
  408. B.addApproval('Code-Review', 2)
  409. # A Depends-On: B
  410. A.editBody('Depends-On: %s\n' % (B.data['url']))
  411. event = A.addLabel('approved')
  412. self.fake_github.emitEvent(event)
  413. self.waitUntilSettled()
  414. self.assertFalse(A.is_merged)
  415. self.assertEqual(B.data['status'], 'NEW')
  416. for connection in self.scheds.first.connections.connections.values():
  417. connection.maintainCache([])
  418. self.executor_server.hold_jobs_in_build = True
  419. B.addApproval('Approved', 1)
  420. self.fake_github.emitEvent(event)
  421. self.waitUntilSettled()
  422. self.executor_server.release('.*-merge')
  423. self.waitUntilSettled()
  424. self.executor_server.release('.*-merge')
  425. self.waitUntilSettled()
  426. self.executor_server.hold_jobs_in_build = False
  427. self.executor_server.release()
  428. self.waitUntilSettled()
  429. self.assertTrue(A.is_merged)
  430. self.assertEqual(B.data['status'], 'MERGED')
  431. self.assertEqual(len(A.comments), 2)
  432. self.assertEqual(B.reported, 2)
  433. changes = self.getJobFromHistory(
  434. 'project-merge', 'github/project2').changes
  435. self.assertEqual(changes, '1,1 1,%s' % A.head_sha)
  436. def test_crd_branch(self):
  437. "Test cross-repo dependencies in multiple branches"
  438. self.create_branch('gerrit/project1', 'mp')
  439. A = self.fake_github.openFakePullRequest('github/project2', 'master',
  440. 'A')
  441. B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B')
  442. C1 = self.fake_gerrit.addFakeChange('gerrit/project1', 'mp', 'C1')
  443. B.addApproval('Code-Review', 2)
  444. C1.addApproval('Code-Review', 2)
  445. # A Depends-On: B+C1
  446. A.editBody('Depends-On: %s\nDepends-On: %s\n' % (
  447. B.data['url'], C1.data['url']))
  448. self.executor_server.hold_jobs_in_build = True
  449. B.addApproval('Approved', 1)
  450. C1.addApproval('Approved', 1)
  451. self.fake_github.emitEvent(A.addLabel('approved'))
  452. self.waitUntilSettled()
  453. self.executor_server.release('.*-merge')
  454. self.waitUntilSettled()
  455. self.executor_server.release('.*-merge')
  456. self.waitUntilSettled()
  457. self.executor_server.release('.*-merge')
  458. self.waitUntilSettled()
  459. self.executor_server.hold_jobs_in_build = False
  460. self.executor_server.release()
  461. self.waitUntilSettled()
  462. self.assertTrue(A.is_merged)
  463. self.assertEqual(B.data['status'], 'MERGED')
  464. self.assertEqual(C1.data['status'], 'MERGED')
  465. self.assertEqual(len(A.comments), 2)
  466. self.assertEqual(B.reported, 2)
  467. self.assertEqual(C1.reported, 2)
  468. changes = self.getJobFromHistory(
  469. 'project-merge', 'github/project2').changes
  470. self.assertEqual(changes, '1,1 2,1 1,%s' %
  471. (A.head_sha,))
  472. def test_crd_gate_reverse(self):
  473. "Test reverse cross-repo dependencies"
  474. A = self.fake_github.openFakePullRequest('github/project2', 'master',
  475. 'A')
  476. B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B')
  477. B.addApproval('Code-Review', 2)
  478. # A Depends-On: B
  479. A.editBody('Depends-On: %s\n' % (B.data['url'],))
  480. self.fake_github.emitEvent(A.addLabel('approved'))
  481. self.waitUntilSettled()
  482. self.assertFalse(A.is_merged)
  483. self.assertEqual(B.data['status'], 'NEW')
  484. self.executor_server.hold_jobs_in_build = True
  485. A.addLabel('approved')
  486. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  487. self.waitUntilSettled()
  488. self.executor_server.release('.*-merge')
  489. self.waitUntilSettled()
  490. self.executor_server.release('.*-merge')
  491. self.waitUntilSettled()
  492. self.executor_server.hold_jobs_in_build = False
  493. self.executor_server.release()
  494. self.waitUntilSettled()
  495. self.assertTrue(A.is_merged)
  496. self.assertEqual(B.data['status'], 'MERGED')
  497. self.assertEqual(len(A.comments), 2)
  498. self.assertEqual(B.reported, 2)
  499. changes = self.getJobFromHistory(
  500. 'project-merge', 'github/project2').changes
  501. self.assertEqual(changes, '1,1 1,%s' %
  502. (A.head_sha,))
  503. def test_crd_cycle(self):
  504. "Test cross-repo dependency cycles"
  505. A = self.fake_github.openFakePullRequest('github/project2', 'master',
  506. 'A')
  507. B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B')
  508. B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  509. B.subject, A.url)
  510. B.addApproval('Code-Review', 2)
  511. B.addApproval('Approved', 1)
  512. # A -> B -> A (via commit-depends)
  513. A.editBody('Depends-On: %s\n' % (B.data['url'],))
  514. self.fake_github.emitEvent(A.addLabel('approved'))
  515. self.waitUntilSettled()
  516. self.assertEqual(len(A.comments), 1)
  517. self.assertEqual(B.reported, 0)
  518. self.assertFalse(A.is_merged)
  519. self.assertEqual(B.data['status'], 'NEW')
  520. def test_crd_gate_unknown(self):
  521. "Test unknown projects in dependent pipeline"
  522. self.init_repo("gerrit/unknown", tag='init')
  523. A = self.fake_github.openFakePullRequest('github/project2', 'master',
  524. 'A')
  525. B = self.fake_gerrit.addFakeChange('gerrit/unknown', 'master', 'B')
  526. B.addApproval('Code-Review', 2)
  527. # A Depends-On: B
  528. A.editBody('Depends-On: %s\n' % (B.data['url'],))
  529. B.addApproval('Approved', 1)
  530. event = A.addLabel('approved')
  531. self.fake_github.emitEvent(event)
  532. self.waitUntilSettled()
  533. # Unknown projects cannot share a queue with any other
  534. # since they don't have common jobs with any other (they have no jobs).
  535. # Changes which depend on unknown project changes
  536. # should not be processed in dependent pipeline
  537. self.assertFalse(A.is_merged)
  538. self.assertEqual(B.data['status'], 'NEW')
  539. self.assertEqual(len(A.comments), 0)
  540. self.assertEqual(B.reported, 0)
  541. self.assertEqual(len(self.history), 0)
  542. # Simulate change B being gated outside this layout Set the
  543. # change merged before submitting the event so that when the
  544. # event triggers a gerrit query to update the change, we get
  545. # the information that it was merged.
  546. B.setMerged()
  547. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  548. self.waitUntilSettled()
  549. self.assertEqual(len(self.history), 0)
  550. # Now that B is merged, A should be able to be enqueued and
  551. # merged.
  552. self.fake_github.emitEvent(event)
  553. self.waitUntilSettled()
  554. self.assertTrue(A.is_merged)
  555. self.assertEqual(len(A.comments), 2)
  556. self.assertEqual(B.data['status'], 'MERGED')
  557. self.assertEqual(B.reported, 0)
  558. def test_crd_check(self):
  559. "Test cross-repo dependencies in independent pipelines"
  560. self.executor_server.hold_jobs_in_build = True
  561. self.gearman_server.hold_jobs_in_queue = True
  562. A = self.fake_github.openFakePullRequest('github/project2', 'master',
  563. 'A')
  564. B = self.fake_gerrit.addFakeChange(
  565. 'gerrit/project1', 'master', 'B')
  566. # A Depends-On: B
  567. A.editBody('Depends-On: %s\n' % (B.data['url'],))
  568. self.fake_github.emitEvent(A.getPullRequestEditedEvent())
  569. self.waitUntilSettled()
  570. self.gearman_server.hold_jobs_in_queue = False
  571. self.gearman_server.release()
  572. self.waitUntilSettled()
  573. self.executor_server.release('.*-merge')
  574. self.waitUntilSettled()
  575. self.assertTrue(self.builds[0].hasChanges(A, B))
  576. self.executor_server.hold_jobs_in_build = False
  577. self.executor_server.release()
  578. self.waitUntilSettled()
  579. self.assertFalse(A.is_merged)
  580. self.assertEqual(B.data['status'], 'NEW')
  581. self.assertEqual(len(A.comments), 1)
  582. self.assertEqual(B.reported, 0)
  583. changes = self.getJobFromHistory(
  584. 'project-merge', 'github/project2').changes
  585. self.assertEqual(changes, '1,1 1,%s' %
  586. (A.head_sha,))
  587. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  588. self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
  589. def test_crd_check_duplicate(self):
  590. "Test duplicate check in independent pipelines"
  591. self.executor_server.hold_jobs_in_build = True
  592. A = self.fake_github.openFakePullRequest('github/project2', 'master',
  593. 'A')
  594. B = self.fake_gerrit.addFakeChange(
  595. 'gerrit/project1', 'master', 'B')
  596. # A Depends-On: B
  597. A.editBody('Depends-On: %s\n' % (B.data['url'],))
  598. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  599. check_pipeline = tenant.layout.pipelines['check']
  600. # Add two dependent changes...
  601. self.fake_github.emitEvent(A.getPullRequestEditedEvent())
  602. self.waitUntilSettled()
  603. self.assertEqual(len(check_pipeline.getAllItems()), 2)
  604. # ...make sure the live one is not duplicated...
  605. self.fake_github.emitEvent(A.getPullRequestEditedEvent())
  606. self.waitUntilSettled()
  607. self.assertEqual(len(check_pipeline.getAllItems()), 2)
  608. # ...but the non-live one is able to be.
  609. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  610. self.waitUntilSettled()
  611. self.assertEqual(len(check_pipeline.getAllItems()), 3)
  612. # Release jobs in order to avoid races with change A jobs
  613. # finishing before change B jobs.
  614. self.orderedRelease()
  615. self.executor_server.hold_jobs_in_build = False
  616. self.executor_server.release()
  617. self.waitUntilSettled()
  618. self.assertFalse(A.is_merged)
  619. self.assertEqual(B.data['status'], 'NEW')
  620. self.assertEqual(len(A.comments), 1)
  621. self.assertEqual(B.reported, 1)
  622. changes = self.getJobFromHistory(
  623. 'project-merge', 'github/project2').changes
  624. self.assertEqual(changes, '1,1 1,%s' %
  625. (A.head_sha,))
  626. changes = self.getJobFromHistory(
  627. 'project-merge', 'gerrit/project1').changes
  628. self.assertEqual(changes, '1,1')
  629. self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
  630. self.assertIn('Build succeeded', A.comments[0])
  631. def _test_crd_check_reconfiguration(self, project1, project2):
  632. "Test cross-repo dependencies re-enqueued in independent pipelines"
  633. self.gearman_server.hold_jobs_in_queue = True
  634. A = self.fake_github.openFakePullRequest('github/project2', 'master',
  635. 'A')
  636. B = self.fake_gerrit.addFakeChange(
  637. 'gerrit/project1', 'master', 'B')
  638. # A Depends-On: B
  639. A.editBody('Depends-On: %s\n' % (B.data['url'],))
  640. self.fake_github.emitEvent(A.getPullRequestEditedEvent())
  641. self.waitUntilSettled()
  642. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  643. # Make sure the items still share a change queue, and the
  644. # first one is not live.
  645. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  646. self.assertEqual(len(tenant.layout.pipelines['check'].queues), 1)
  647. queue = tenant.layout.pipelines['check'].queues[0]
  648. first_item = queue.queue[0]
  649. for item in queue.queue:
  650. self.assertEqual(item.queue, first_item.queue)
  651. self.assertFalse(first_item.live)
  652. self.assertTrue(queue.queue[1].live)
  653. self.gearman_server.hold_jobs_in_queue = False
  654. self.gearman_server.release()
  655. self.waitUntilSettled()
  656. self.assertFalse(A.is_merged)
  657. self.assertEqual(B.data['status'], 'NEW')
  658. self.assertEqual(len(A.comments), 1)
  659. self.assertEqual(B.reported, 0)
  660. changes = self.getJobFromHistory(
  661. 'project-merge', 'github/project2').changes
  662. self.assertEqual(changes, '1,1 1,%s' %
  663. (A.head_sha,))
  664. self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
  665. def test_crd_check_reconfiguration(self):
  666. self._test_crd_check_reconfiguration('org/project1', 'org/project2')
  667. def test_crd_undefined_project(self):
  668. """Test that undefined projects in dependencies are handled for
  669. independent pipelines"""
  670. # It's a hack for fake gerrit,
  671. # as it implies repo creation upon the creation of any change
  672. self.init_repo("gerrit/unknown", tag='init')
  673. self._test_crd_check_reconfiguration('github/project2',
  674. 'gerrit/unknown')
  675. def test_crd_check_transitive(self):
  676. "Test transitive cross-repo dependencies"
  677. # Specifically, if A -> B -> C, and C gets a new patchset and
  678. # A gets a new patchset, ensure the test of A,2 includes B,1
  679. # and C,2 (not C,1 which would indicate stale data in the
  680. # cache for B).
  681. A = self.fake_github.openFakePullRequest('github/project2', 'master',
  682. 'A')
  683. B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B')
  684. C = self.fake_github.openFakePullRequest('github/project2', 'master',
  685. 'C')
  686. # B Depends-On: C
  687. B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  688. B.subject, C.url)
  689. # A Depends-On: B
  690. A.editBody('Depends-On: %s\n' % (B.data['url'],))
  691. self.fake_github.emitEvent(A.getPullRequestEditedEvent())
  692. self.waitUntilSettled()
  693. self.assertEqual(self.history[-1].changes, '2,%s 1,1 1,%s' %
  694. (C.head_sha, A.head_sha))
  695. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  696. self.waitUntilSettled()
  697. self.assertEqual(self.history[-1].changes, '2,%s 1,1' %
  698. (C.head_sha,))
  699. self.fake_github.emitEvent(C.getPullRequestEditedEvent())
  700. self.waitUntilSettled()
  701. self.assertEqual(self.history[-1].changes, '2,%s' %
  702. (C.head_sha,))
  703. new_c_head = C.head_sha
  704. C.addCommit()
  705. old_c_head = C.head_sha
  706. self.assertNotEqual(old_c_head, new_c_head)
  707. self.fake_github.emitEvent(C.getPullRequestEditedEvent())
  708. self.waitUntilSettled()
  709. self.assertEqual(self.history[-1].changes, '2,%s' %
  710. (C.head_sha,))
  711. new_a_head = A.head_sha
  712. A.addCommit()
  713. old_a_head = A.head_sha
  714. self.assertNotEqual(old_a_head, new_a_head)
  715. self.fake_github.emitEvent(A.getPullRequestEditedEvent())
  716. self.waitUntilSettled()
  717. self.assertEqual(self.history[-1].changes, '2,%s 1,1 1,%s' %
  718. (C.head_sha, A.head_sha,))
  719. def test_crd_check_unknown(self):
  720. "Test unknown projects in independent pipeline"
  721. self.init_repo("gerrit/unknown", tag='init')
  722. A = self.fake_github.openFakePullRequest('github/project2', 'master',
  723. 'A')
  724. B = self.fake_gerrit.addFakeChange(
  725. 'gerrit/unknown', 'master', 'B')
  726. # A Depends-On: B
  727. A.editBody('Depends-On: %s\n' % (B.data['url'],))
  728. # Make sure zuul has seen an event on B.
  729. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  730. # Note we wait until settled here as the event processing for
  731. # the next event may not have the updated db yet otherwise.
  732. self.waitUntilSettled()
  733. self.fake_github.emitEvent(A.getPullRequestEditedEvent())
  734. self.waitUntilSettled()
  735. self.assertFalse(A.is_merged)
  736. self.assertEqual(len(A.comments), 1)
  737. self.assertEqual(B.data['status'], 'NEW')
  738. self.assertEqual(B.reported, 0)
  739. def test_crd_cycle_join(self):
  740. "Test an updated change creates a cycle"
  741. A = self.fake_gerrit.addFakeChange(
  742. 'gerrit/project1', 'master', 'A')
  743. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  744. self.waitUntilSettled()
  745. self.assertEqual(A.reported, 1)
  746. # Create B->A
  747. B = self.fake_github.openFakePullRequest('github/project2', 'master',
  748. 'B')
  749. B.editBody('Depends-On: %s\n' % (A.data['url'],))
  750. self.fake_github.emitEvent(B.getPullRequestEditedEvent())
  751. self.waitUntilSettled()
  752. # Dep is there so zuul should have reported on B
  753. self.assertEqual(len(B.comments), 1)
  754. # Update A to add A->B (a cycle).
  755. A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  756. A.subject, B.url)
  757. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  758. self.waitUntilSettled()
  759. # Dependency cycle injected so zuul should have reported again on A
  760. self.assertEqual(A.reported, 2)
  761. # Now if we update B to remove the depends-on, everything
  762. # should be okay. B; A->B
  763. B.addCommit()
  764. B.editBody('')
  765. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  766. self.waitUntilSettled()
  767. # Cycle was removed so now zuul should have reported again on A
  768. self.assertEqual(A.reported, 3)
  769. self.fake_github.emitEvent(B.getPullRequestEditedEvent())
  770. self.waitUntilSettled()
  771. self.assertEqual(len(B.comments), 2)