The Gatekeeper, or a project gating system
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

8436 lines
344 KiB

  1. # Copyright 2012 Hewlett-Packard Development Company, L.P.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  4. # not use this file except in compliance with the License. You may obtain
  5. # a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12. # License for the specific language governing permissions and limitations
  13. # under the License.
  14. import gc
  15. import json
  16. import textwrap
  17. import os
  18. import shutil
  19. import socket
  20. import time
  21. from unittest import mock
  22. from unittest import skip
  23. from kazoo.exceptions import NoNodeError
  24. import git
  25. import testtools
  26. from zuul.scheduler import Scheduler
  27. import zuul.change_matcher
  28. from zuul.driver.gerrit import gerritreporter
  29. import zuul.scheduler
  30. import zuul.rpcclient
  31. import zuul.model
  32. from tests.base import (
  33. SSLZuulTestCase,
  34. ZuulTestCase,
  35. repack_repo,
  36. simple_layout,
  37. iterate_timeout,
  38. )
  39. class TestSchedulerSSL(SSLZuulTestCase):
  40. tenant_config_file = 'config/single-tenant/main.yaml'
  41. def test_jobs_executed(self):
  42. "Test that jobs are executed and a change is merged"
  43. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  44. A.addApproval('Code-Review', 2)
  45. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  46. self.waitUntilSettled()
  47. self.assertEqual(self.getJobFromHistory('project-merge').result,
  48. 'SUCCESS')
  49. self.assertEqual(self.getJobFromHistory('project-test1').result,
  50. 'SUCCESS')
  51. self.assertEqual(self.getJobFromHistory('project-test2').result,
  52. 'SUCCESS')
  53. self.assertEqual(A.data['status'], 'MERGED')
  54. self.assertEqual(A.reported, 2)
  55. self.assertEqual(self.getJobFromHistory('project-test1').node,
  56. 'label1')
  57. self.assertEqual(self.getJobFromHistory('project-test2').node,
  58. 'label1')
  59. class TestSchedulerZone(ZuulTestCase):
  60. tenant_config_file = 'config/single-tenant/main.yaml'
  61. def setUp(self):
  62. super(TestSchedulerZone, self).setUp()
  63. self.fake_nodepool.attributes = {'executor-zone': 'test-provider.vpn'}
  64. def setup_config(self, config_file: str):
  65. config = super(TestSchedulerZone, self).setup_config(config_file)
  66. config.set('executor', 'zone', 'test-provider.vpn')
  67. return config
  68. def test_jobs_executed(self):
  69. "Test that jobs are executed and a change is merged per zone"
  70. self.gearman_server.hold_jobs_in_queue = True
  71. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  72. A.addApproval('Code-Review', 2)
  73. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  74. self.waitUntilSettled()
  75. queue = self.gearman_server.getQueue()
  76. self.assertEqual(len(self.builds), 0)
  77. self.assertEqual(len(queue), 1)
  78. self.assertEqual(b'executor:execute:test-provider.vpn', queue[0].name)
  79. self.gearman_server.hold_jobs_in_queue = False
  80. self.gearman_server.release()
  81. self.waitUntilSettled()
  82. self.assertEqual(self.getJobFromHistory('project-merge').result,
  83. 'SUCCESS')
  84. self.assertEqual(self.getJobFromHistory('project-test1').result,
  85. 'SUCCESS')
  86. self.assertEqual(self.getJobFromHistory('project-test2').result,
  87. 'SUCCESS')
  88. self.assertEqual(A.data['status'], 'MERGED')
  89. self.assertEqual(A.reported, 2)
  90. self.assertEqual(self.getJobFromHistory('project-test1').node,
  91. 'label1')
  92. self.assertEqual(self.getJobFromHistory('project-test2').node,
  93. 'label1')
  94. class TestAuthorizeViaRPC(ZuulTestCase):
  95. tenant_config_file = 'config/authorization/single-tenant/main.yaml'
  96. def test_authorize_via_rpc(self):
  97. client = zuul.rpcclient.RPCClient('127.0.0.1',
  98. self.gearman_server.port)
  99. self.addCleanup(client.shutdown)
  100. claims = {'__zuul_uid_claim': 'venkman'}
  101. authorized = client.submitJob('zuul:authorize_user',
  102. {'tenant': 'tenant-one',
  103. 'claims': claims}).data[0]
  104. self.assertTrue(json.loads(authorized))
  105. claims = {'sub': 'gozer'}
  106. authorized = client.submitJob('zuul:authorize_user',
  107. {'tenant': 'tenant-one',
  108. 'claims': claims}).data[0]
  109. self.assertTrue(not json.loads(authorized))
  110. claims = {'sub': 'stantz',
  111. 'iss': 'columbia.edu'}
  112. authorized = client.submitJob('zuul:authorize_user',
  113. {'tenant': 'tenant-one',
  114. 'claims': claims}).data[0]
  115. self.assertTrue(json.loads(authorized))
  116. claims = {'sub': 'slimer',
  117. 'groups': ['ghostbusters', 'ectoplasms']}
  118. authorized = client.submitJob('zuul:authorize_user',
  119. {'tenant': 'tenant-one',
  120. 'claims': claims}).data[0]
  121. self.assertTrue(json.loads(authorized))
  122. class TestAuthorizeWithTemplatingViaRPC(ZuulTestCase):
  123. tenant_config_file = 'config/authorization/rules-templating/main.yaml'
  124. def test_authorize_via_rpc(self):
  125. client = zuul.rpcclient.RPCClient('127.0.0.1',
  126. self.gearman_server.port)
  127. self.addCleanup(client.shutdown)
  128. tenants = ['tenant-zero', 'tenant-one', 'tenant-two']
  129. for t_claim in tenants:
  130. claims = {'groups': [t_claim, ]}
  131. for tenant in tenants:
  132. authorized = client.submitJob('zuul:authorize_user',
  133. {'tenant': tenant,
  134. 'claims': claims}).data[0]
  135. if t_claim == tenant:
  136. self.assertTrue(
  137. json.loads(authorized),
  138. "Failed for t_claim: %s, tenant: %s" % (t_claim,
  139. tenant))
  140. else:
  141. self.assertTrue(
  142. not json.loads(authorized),
  143. "Failed for t_claim: %s, tenant: %s" % (t_claim,
  144. tenant))
  145. class TestSchedulerAutoholdHoldExpiration(ZuulTestCase):
  146. '''
  147. This class of tests validates the autohold node expiration values
  148. are set correctly via zuul config or from a custom value.
  149. '''
  150. config_file = 'zuul-hold-expiration.conf'
  151. tenant_config_file = 'config/single-tenant/main.yaml'
  152. @simple_layout('layouts/autohold.yaml')
  153. def test_autohold_max_hold_default(self):
  154. '''
  155. Test that the hold request node expiration will default to the
  156. value specified in the configuration file.
  157. '''
  158. client = zuul.rpcclient.RPCClient('127.0.0.1',
  159. self.gearman_server.port)
  160. self.addCleanup(client.shutdown)
  161. # Add a autohold with no hold expiration.
  162. r = client.autohold('tenant-one', 'org/project', 'project-test2',
  163. "", "", "reason text", 1)
  164. self.assertTrue(r)
  165. # There should be a record in ZooKeeper
  166. request_list = self.scheds.first.sched.zk.getHoldRequests()
  167. self.assertEqual(1, len(request_list))
  168. request = self.scheds.first.sched.zk.getHoldRequest(request_list[0])
  169. self.assertIsNotNone(request)
  170. self.assertEqual('tenant-one', request.tenant)
  171. self.assertEqual('review.example.com/org/project', request.project)
  172. self.assertEqual('project-test2', request.job)
  173. self.assertEqual('reason text', request.reason)
  174. self.assertEqual(1, request.max_count)
  175. self.assertEqual(0, request.current_count)
  176. self.assertEqual([], request.nodes)
  177. # This should be the default value from the zuul config file.
  178. self.assertEqual(1800, request.node_expiration)
  179. @simple_layout('layouts/autohold.yaml')
  180. def test_autohold_max_hold_custom(self):
  181. '''
  182. Test that the hold request node expiration will be set to the custom
  183. value specified in the request.
  184. '''
  185. client = zuul.rpcclient.RPCClient('127.0.0.1',
  186. self.gearman_server.port)
  187. self.addCleanup(client.shutdown)
  188. # Add a autohold with a custom hold expiration.
  189. r = client.autohold('tenant-one', 'org/project', 'project-test2',
  190. "", "", "reason text", 1, 500)
  191. self.assertTrue(r)
  192. # There should be a record in ZooKeeper
  193. request_list = self.scheds.first.sched.zk.getHoldRequests()
  194. self.assertEqual(1, len(request_list))
  195. request = self.scheds.first.sched.zk.getHoldRequest(request_list[0])
  196. self.assertIsNotNone(request)
  197. self.assertEqual('tenant-one', request.tenant)
  198. self.assertEqual('review.example.com/org/project', request.project)
  199. self.assertEqual('project-test2', request.job)
  200. self.assertEqual('reason text', request.reason)
  201. self.assertEqual(1, request.max_count)
  202. self.assertEqual(0, request.current_count)
  203. self.assertEqual([], request.nodes)
  204. # This should be the value from the user request.
  205. self.assertEqual(500, request.node_expiration)
  206. @simple_layout('layouts/autohold.yaml')
  207. def test_autohold_max_hold_custom_invalid(self):
  208. '''
  209. Test that if the custom hold request node expiration is higher than our
  210. configured max, it will be lowered to the max.
  211. '''
  212. client = zuul.rpcclient.RPCClient('127.0.0.1',
  213. self.gearman_server.port)
  214. self.addCleanup(client.shutdown)
  215. # Add a autohold with a custom hold expiration that is higher than our
  216. # configured max.
  217. r = client.autohold('tenant-one', 'org/project', 'project-test2',
  218. "", "", "reason text", 1, 10000)
  219. self.assertTrue(r)
  220. # There should be a record in ZooKeeper
  221. request_list = self.scheds.first.sched.zk.getHoldRequests()
  222. self.assertEqual(1, len(request_list))
  223. request = self.scheds.first.sched.zk.getHoldRequest(request_list[0])
  224. self.assertIsNotNone(request)
  225. self.assertEqual('tenant-one', request.tenant)
  226. self.assertEqual('review.example.com/org/project', request.project)
  227. self.assertEqual('project-test2', request.job)
  228. self.assertEqual('reason text', request.reason)
  229. self.assertEqual(1, request.max_count)
  230. self.assertEqual(0, request.current_count)
  231. self.assertEqual([], request.nodes)
  232. # This should be the max value from the zuul config file.
  233. self.assertEqual(3600, request.node_expiration)
  234. class TestScheduler(ZuulTestCase):
  235. tenant_config_file = 'config/single-tenant/main.yaml'
  236. def test_jobs_executed(self):
  237. "Test that jobs are executed and a change is merged"
  238. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  239. A.addApproval('Code-Review', 2)
  240. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  241. self.waitUntilSettled()
  242. self.assertEqual(self.getJobFromHistory('project-merge').result,
  243. 'SUCCESS')
  244. self.assertEqual(self.getJobFromHistory('project-test1').result,
  245. 'SUCCESS')
  246. self.assertEqual(self.getJobFromHistory('project-test2').result,
  247. 'SUCCESS')
  248. self.assertEqual(A.data['status'], 'MERGED')
  249. self.assertEqual(A.reported, 2)
  250. self.assertEqual(self.getJobFromHistory('project-test1').node,
  251. 'label1')
  252. self.assertEqual(self.getJobFromHistory('project-test2').node,
  253. 'label1')
  254. # TODOv3(jeblair): we may want to report stats by tenant (also?).
  255. # Per-driver
  256. self.assertReportedStat('zuul.event.gerrit.comment-added', value='1',
  257. kind='c')
  258. # Per-driver per-connection
  259. self.assertReportedStat('zuul.event.gerrit.gerrit.comment-added',
  260. value='1', kind='c')
  261. self.assertReportedStat(
  262. 'zuul.tenant.tenant-one.pipeline.gate.current_changes',
  263. value='1', kind='g')
  264. self.assertReportedStat(
  265. 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
  266. 'org_project.master.job.project-merge.SUCCESS', kind='ms')
  267. self.assertReportedStat(
  268. 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
  269. 'org_project.master.job.project-merge.SUCCESS', value='1',
  270. kind='c')
  271. self.assertReportedStat(
  272. 'zuul.tenant.tenant-one.pipeline.gate.resident_time', kind='ms')
  273. self.assertReportedStat(
  274. 'zuul.tenant.tenant-one.pipeline.gate.total_changes', value='1',
  275. kind='c')
  276. self.assertReportedStat(
  277. 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
  278. 'org_project.master.resident_time', kind='ms')
  279. self.assertReportedStat(
  280. 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
  281. 'org_project.master.total_changes', value='1', kind='c')
  282. exec_key = 'zuul.executor.%s' % self.executor_server.hostname.replace(
  283. '.', '_')
  284. self.assertReportedStat(exec_key + '.builds', value='1', kind='c')
  285. self.assertReportedStat(exec_key + '.starting_builds', kind='g')
  286. self.assertReportedStat(exec_key + '.starting_builds', kind='ms')
  287. self.assertReportedStat(
  288. 'zuul.nodepool.requests.requested.total', value='1', kind='c')
  289. self.assertReportedStat(
  290. 'zuul.nodepool.requests.requested.label.label1',
  291. value='1', kind='c')
  292. self.assertReportedStat(
  293. 'zuul.nodepool.requests.fulfilled.label.label1',
  294. value='1', kind='c')
  295. self.assertReportedStat(
  296. 'zuul.nodepool.requests.requested.size.1', value='1', kind='c')
  297. self.assertReportedStat(
  298. 'zuul.nodepool.requests.fulfilled.size.1', value='1', kind='c')
  299. self.assertReportedStat(
  300. 'zuul.nodepool.current_requests', value='1', kind='g')
  301. self.assertReportedStat(
  302. 'zuul.executors.online', value='1', kind='g')
  303. self.assertReportedStat(
  304. 'zuul.executors.accepting', value='1', kind='g')
  305. self.assertReportedStat(
  306. 'zuul.mergers.online', value='1', kind='g')
  307. for build in self.history:
  308. self.assertTrue(build.parameters['zuul']['voting'])
  309. def test_initial_pipeline_gauges(self):
  310. "Test that each pipeline reported its length on start"
  311. self.assertReportedStat('zuul.tenant.tenant-one.pipeline.gate.'
  312. 'current_changes',
  313. value='0', kind='g')
  314. self.assertReportedStat('zuul.tenant.tenant-one.pipeline.check.'
  315. 'current_changes',
  316. value='0', kind='g')
  317. def test_job_branch(self):
  318. "Test the correct variant of a job runs on a branch"
  319. self.create_branch('org/project', 'stable')
  320. self.fake_gerrit.addEvent(
  321. self.fake_gerrit.getFakeBranchCreatedEvent(
  322. 'org/project', 'stable'))
  323. self.waitUntilSettled()
  324. A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A')
  325. A.addApproval('Code-Review', 2)
  326. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  327. self.waitUntilSettled()
  328. self.assertEqual(self.getJobFromHistory('project-test1').result,
  329. 'SUCCESS')
  330. self.assertEqual(self.getJobFromHistory('project-test2').result,
  331. 'SUCCESS')
  332. self.assertEqual(A.data['status'], 'MERGED')
  333. self.assertEqual(A.reported, 2,
  334. "A should report start and success")
  335. self.assertIn('gate', A.messages[1],
  336. "A should transit gate")
  337. self.assertEqual(self.getJobFromHistory('project-test1').node,
  338. 'label2')
  339. @simple_layout('layouts/branch-deletion.yaml')
  340. def test_branch_deletion(self):
  341. "Test the correct variant of a job runs on a branch"
  342. self._startMerger()
  343. merger_gear = self.executor_server.merger_gearworker.gearman
  344. for f in list(merger_gear.functions.keys()):
  345. f = f.decode('utf8')
  346. if f.startswith('merger:'):
  347. merger_gear.unRegisterFunction(f)
  348. self.create_branch('org/project', 'stable')
  349. self.fake_gerrit.addEvent(
  350. self.fake_gerrit.getFakeBranchCreatedEvent(
  351. 'org/project', 'stable'))
  352. self.waitUntilSettled()
  353. A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A')
  354. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  355. self.waitUntilSettled()
  356. self.assertEqual(self.getJobFromHistory('project-test2').result,
  357. 'SUCCESS')
  358. self.delete_branch('org/project', 'stable')
  359. path = os.path.join(self.executor_src_root, 'review.example.com')
  360. shutil.rmtree(path)
  361. self.executor_server.hold_jobs_in_build = True
  362. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  363. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  364. self.waitUntilSettled()
  365. build = self.builds[0]
  366. # Make sure there is no stable branch in the checked out git repo.
  367. pname = 'review.example.com/org/project'
  368. work = build.getWorkspaceRepos([pname])
  369. work = work[pname]
  370. heads = set([str(x) for x in work.heads])
  371. self.assertEqual(heads, set(['master']))
  372. self.executor_server.hold_jobs_in_build = False
  373. build.release()
  374. self.waitUntilSettled()
  375. self.assertEqual(self.getJobFromHistory('project-test1').result,
  376. 'SUCCESS')
  377. def test_parallel_changes(self):
  378. "Test that changes are tested in parallel and merged in series"
  379. self.executor_server.hold_jobs_in_build = True
  380. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  381. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  382. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  383. A.addApproval('Code-Review', 2)
  384. B.addApproval('Code-Review', 2)
  385. C.addApproval('Code-Review', 2)
  386. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  387. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  388. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  389. self.waitUntilSettled()
  390. self.assertEqual(len(self.builds), 1)
  391. self.assertEqual(self.builds[0].name, 'project-merge')
  392. self.assertTrue(self.builds[0].hasChanges(A))
  393. self.executor_server.release('.*-merge')
  394. self.waitUntilSettled()
  395. self.assertEqual(len(self.builds), 3)
  396. self.assertEqual(self.builds[0].name, 'project-test1')
  397. self.assertTrue(self.builds[0].hasChanges(A))
  398. self.assertEqual(self.builds[1].name, 'project-test2')
  399. self.assertTrue(self.builds[1].hasChanges(A))
  400. self.assertEqual(self.builds[2].name, 'project-merge')
  401. self.assertTrue(self.builds[2].hasChanges(A, B))
  402. self.executor_server.release('.*-merge')
  403. self.waitUntilSettled()
  404. self.assertEqual(len(self.builds), 5)
  405. self.assertEqual(self.builds[0].name, 'project-test1')
  406. self.assertTrue(self.builds[0].hasChanges(A))
  407. self.assertEqual(self.builds[1].name, 'project-test2')
  408. self.assertTrue(self.builds[1].hasChanges(A))
  409. self.assertEqual(self.builds[2].name, 'project-test1')
  410. self.assertTrue(self.builds[2].hasChanges(A, B))
  411. self.assertEqual(self.builds[3].name, 'project-test2')
  412. self.assertTrue(self.builds[3].hasChanges(A, B))
  413. self.assertEqual(self.builds[4].name, 'project-merge')
  414. self.assertTrue(self.builds[4].hasChanges(A, B, C))
  415. self.executor_server.release('.*-merge')
  416. self.waitUntilSettled()
  417. self.assertEqual(len(self.builds), 6)
  418. self.assertEqual(self.builds[0].name, 'project-test1')
  419. self.assertTrue(self.builds[0].hasChanges(A))
  420. self.assertEqual(self.builds[1].name, 'project-test2')
  421. self.assertTrue(self.builds[1].hasChanges(A))
  422. self.assertEqual(self.builds[2].name, 'project-test1')
  423. self.assertTrue(self.builds[2].hasChanges(A, B))
  424. self.assertEqual(self.builds[3].name, 'project-test2')
  425. self.assertTrue(self.builds[3].hasChanges(A, B))
  426. self.assertEqual(self.builds[4].name, 'project-test1')
  427. self.assertTrue(self.builds[4].hasChanges(A, B, C))
  428. self.assertEqual(self.builds[5].name, 'project-test2')
  429. self.assertTrue(self.builds[5].hasChanges(A, B, C))
  430. self.executor_server.hold_jobs_in_build = False
  431. self.executor_server.release()
  432. self.waitUntilSettled()
  433. self.assertEqual(len(self.builds), 0)
  434. self.assertEqual(len(self.history), 9)
  435. self.assertEqual(A.data['status'], 'MERGED')
  436. self.assertEqual(B.data['status'], 'MERGED')
  437. self.assertEqual(C.data['status'], 'MERGED')
  438. self.assertEqual(A.reported, 2)
  439. self.assertEqual(B.reported, 2)
  440. self.assertEqual(C.reported, 2)
  441. def test_failed_changes(self):
  442. "Test that a change behind a failed change is retested"
  443. self.executor_server.hold_jobs_in_build = True
  444. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  445. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  446. A.addApproval('Code-Review', 2)
  447. B.addApproval('Code-Review', 2)
  448. self.executor_server.failJob('project-test1', A)
  449. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  450. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  451. self.waitUntilSettled()
  452. self.assertBuilds([dict(name='project-merge', changes='1,1')])
  453. self.executor_server.release('.*-merge')
  454. self.waitUntilSettled()
  455. # A/project-merge is complete
  456. self.assertBuilds([
  457. dict(name='project-test1', changes='1,1'),
  458. dict(name='project-test2', changes='1,1'),
  459. dict(name='project-merge', changes='1,1 2,1'),
  460. ])
  461. self.executor_server.release('.*-merge')
  462. self.waitUntilSettled()
  463. # A/project-merge is complete
  464. # B/project-merge is complete
  465. self.assertBuilds([
  466. dict(name='project-test1', changes='1,1'),
  467. dict(name='project-test2', changes='1,1'),
  468. dict(name='project-test1', changes='1,1 2,1'),
  469. dict(name='project-test2', changes='1,1 2,1'),
  470. ])
  471. # Release project-test1 for A which will fail. This will
  472. # abort both running B jobs and reexecute project-merge for B.
  473. self.builds[0].release()
  474. self.waitUntilSettled()
  475. self.orderedRelease()
  476. self.assertHistory([
  477. dict(name='project-merge', result='SUCCESS', changes='1,1'),
  478. dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
  479. dict(name='project-test1', result='FAILURE', changes='1,1'),
  480. dict(name='project-test1', result='ABORTED', changes='1,1 2,1'),
  481. dict(name='project-test2', result='ABORTED', changes='1,1 2,1'),
  482. dict(name='project-test2', result='SUCCESS', changes='1,1'),
  483. dict(name='project-merge', result='SUCCESS', changes='2,1'),
  484. dict(name='project-test1', result='SUCCESS', changes='2,1'),
  485. dict(name='project-test2', result='SUCCESS', changes='2,1'),
  486. ], ordered=False)
  487. self.assertEqual(A.data['status'], 'NEW')
  488. self.assertEqual(B.data['status'], 'MERGED')
  489. self.assertEqual(A.reported, 2)
  490. self.assertEqual(B.reported, 2)
  491. def test_independent_queues(self):
  492. "Test that changes end up in the right queues"
  493. self.executor_server.hold_jobs_in_build = True
  494. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  495. B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
  496. C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
  497. A.addApproval('Code-Review', 2)
  498. B.addApproval('Code-Review', 2)
  499. C.addApproval('Code-Review', 2)
  500. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  501. self.waitUntilSettled()
  502. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  503. self.waitUntilSettled()
  504. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  505. self.waitUntilSettled()
  506. # There should be one merge job at the head of each queue running
  507. self.assertBuilds([
  508. dict(name='project-merge', changes='1,1'),
  509. dict(name='project-merge', changes='2,1'),
  510. ])
  511. # Release the current merge builds
  512. self.builds[0].release()
  513. self.waitUntilSettled()
  514. self.builds[0].release()
  515. self.waitUntilSettled()
  516. # Release the merge job for project2 which is behind project1
  517. self.executor_server.release('.*-merge')
  518. self.waitUntilSettled()
  519. # All the test builds should be running:
  520. self.assertBuilds([
  521. dict(name='project-test1', changes='1,1'),
  522. dict(name='project-test2', changes='1,1'),
  523. dict(name='project-test1', changes='2,1'),
  524. dict(name='project-test2', changes='2,1'),
  525. dict(name='project1-project2-integration', changes='2,1'),
  526. dict(name='project-test1', changes='2,1 3,1'),
  527. dict(name='project-test2', changes='2,1 3,1'),
  528. dict(name='project1-project2-integration', changes='2,1 3,1'),
  529. ])
  530. self.orderedRelease()
  531. self.assertHistory([
  532. dict(name='project-merge', result='SUCCESS', changes='1,1'),
  533. dict(name='project-merge', result='SUCCESS', changes='2,1'),
  534. dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'),
  535. dict(name='project-test1', result='SUCCESS', changes='1,1'),
  536. dict(name='project-test2', result='SUCCESS', changes='1,1'),
  537. dict(name='project-test1', result='SUCCESS', changes='2,1'),
  538. dict(name='project-test2', result='SUCCESS', changes='2,1'),
  539. dict(
  540. name='project1-project2-integration',
  541. result='SUCCESS',
  542. changes='2,1'),
  543. dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'),
  544. dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'),
  545. dict(name='project1-project2-integration',
  546. result='SUCCESS',
  547. changes='2,1 3,1'),
  548. ])
  549. self.assertEqual(A.data['status'], 'MERGED')
  550. self.assertEqual(B.data['status'], 'MERGED')
  551. self.assertEqual(C.data['status'], 'MERGED')
  552. self.assertEqual(A.reported, 2)
  553. self.assertEqual(B.reported, 2)
  554. self.assertEqual(C.reported, 2)
  555. def test_failed_change_at_head(self):
  556. "Test that if a change at the head fails, jobs behind it are canceled"
  557. self.executor_server.hold_jobs_in_build = True
  558. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  559. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  560. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  561. A.addApproval('Code-Review', 2)
  562. B.addApproval('Code-Review', 2)
  563. C.addApproval('Code-Review', 2)
  564. self.executor_server.failJob('project-test1', A)
  565. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  566. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  567. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  568. self.waitUntilSettled()
  569. self.assertBuilds([
  570. dict(name='project-merge', changes='1,1'),
  571. ])
  572. self.executor_server.release('.*-merge')
  573. self.waitUntilSettled()
  574. self.executor_server.release('.*-merge')
  575. self.waitUntilSettled()
  576. self.executor_server.release('.*-merge')
  577. self.waitUntilSettled()
  578. self.assertBuilds([
  579. dict(name='project-test1', changes='1,1'),
  580. dict(name='project-test2', changes='1,1'),
  581. dict(name='project-test1', changes='1,1 2,1'),
  582. dict(name='project-test2', changes='1,1 2,1'),
  583. dict(name='project-test1', changes='1,1 2,1 3,1'),
  584. dict(name='project-test2', changes='1,1 2,1 3,1'),
  585. ])
  586. self.release(self.builds[0])
  587. self.waitUntilSettled()
  588. # project-test2, project-merge for B
  589. self.assertBuilds([
  590. dict(name='project-test2', changes='1,1'),
  591. dict(name='project-merge', changes='2,1'),
  592. ])
  593. # Unordered history comparison because the aborts can finish
  594. # in any order.
  595. self.assertHistory([
  596. dict(name='project-merge', result='SUCCESS',
  597. changes='1,1'),
  598. dict(name='project-merge', result='SUCCESS',
  599. changes='1,1 2,1'),
  600. dict(name='project-merge', result='SUCCESS',
  601. changes='1,1 2,1 3,1'),
  602. dict(name='project-test1', result='FAILURE',
  603. changes='1,1'),
  604. dict(name='project-test1', result='ABORTED',
  605. changes='1,1 2,1'),
  606. dict(name='project-test2', result='ABORTED',
  607. changes='1,1 2,1'),
  608. dict(name='project-test1', result='ABORTED',
  609. changes='1,1 2,1 3,1'),
  610. dict(name='project-test2', result='ABORTED',
  611. changes='1,1 2,1 3,1'),
  612. ], ordered=False)
  613. self.executor_server.release('.*-merge')
  614. self.waitUntilSettled()
  615. self.executor_server.release('.*-merge')
  616. self.waitUntilSettled()
  617. self.orderedRelease()
  618. self.assertBuilds([])
  619. self.assertHistory([
  620. dict(name='project-merge', result='SUCCESS',
  621. changes='1,1'),
  622. dict(name='project-merge', result='SUCCESS',
  623. changes='1,1 2,1'),
  624. dict(name='project-merge', result='SUCCESS',
  625. changes='1,1 2,1 3,1'),
  626. dict(name='project-test1', result='FAILURE',
  627. changes='1,1'),
  628. dict(name='project-test1', result='ABORTED',
  629. changes='1,1 2,1'),
  630. dict(name='project-test2', result='ABORTED',
  631. changes='1,1 2,1'),
  632. dict(name='project-test1', result='ABORTED',
  633. changes='1,1 2,1 3,1'),
  634. dict(name='project-test2', result='ABORTED',
  635. changes='1,1 2,1 3,1'),
  636. dict(name='project-merge', result='SUCCESS',
  637. changes='2,1'),
  638. dict(name='project-merge', result='SUCCESS',
  639. changes='2,1 3,1'),
  640. dict(name='project-test2', result='SUCCESS',
  641. changes='1,1'),
  642. dict(name='project-test1', result='SUCCESS',
  643. changes='2,1'),
  644. dict(name='project-test2', result='SUCCESS',
  645. changes='2,1'),
  646. dict(name='project-test1', result='SUCCESS',
  647. changes='2,1 3,1'),
  648. dict(name='project-test2', result='SUCCESS',
  649. changes='2,1 3,1'),
  650. ], ordered=False)
  651. self.assertEqual(A.data['status'], 'NEW')
  652. self.assertEqual(B.data['status'], 'MERGED')
  653. self.assertEqual(C.data['status'], 'MERGED')
  654. self.assertEqual(A.reported, 2)
  655. self.assertEqual(B.reported, 2)
  656. self.assertEqual(C.reported, 2)
  657. def test_failed_change_in_middle(self):
  658. "Test a failed change in the middle of the queue"
  659. self.executor_server.hold_jobs_in_build = True
  660. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  661. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  662. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  663. A.addApproval('Code-Review', 2)
  664. B.addApproval('Code-Review', 2)
  665. C.addApproval('Code-Review', 2)
  666. self.executor_server.failJob('project-test1', B)
  667. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  668. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  669. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  670. self.waitUntilSettled()
  671. self.executor_server.release('.*-merge')
  672. self.waitUntilSettled()
  673. self.executor_server.release('.*-merge')
  674. self.waitUntilSettled()
  675. self.executor_server.release('.*-merge')
  676. self.waitUntilSettled()
  677. self.assertEqual(len(self.builds), 6)
  678. self.assertEqual(self.builds[0].name, 'project-test1')
  679. self.assertEqual(self.builds[1].name, 'project-test2')
  680. self.assertEqual(self.builds[2].name, 'project-test1')
  681. self.assertEqual(self.builds[3].name, 'project-test2')
  682. self.assertEqual(self.builds[4].name, 'project-test1')
  683. self.assertEqual(self.builds[5].name, 'project-test2')
  684. self.release(self.builds[2])
  685. self.waitUntilSettled()
  686. # project-test1 and project-test2 for A
  687. # project-test2 for B
  688. # project-merge for C (without B)
  689. self.assertEqual(len(self.builds), 4)
  690. self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 2)
  691. self.executor_server.release('.*-merge')
  692. self.waitUntilSettled()
  693. # project-test1 and project-test2 for A
  694. # project-test2 for B
  695. # project-test1 and project-test2 for C
  696. self.assertEqual(len(self.builds), 5)
  697. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  698. items = tenant.layout.pipelines['gate'].getAllItems()
  699. builds = items[0].current_build_set.getBuilds()
  700. self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
  701. self.assertEqual(self.countJobResults(builds, None), 2)
  702. builds = items[1].current_build_set.getBuilds()
  703. self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
  704. self.assertEqual(self.countJobResults(builds, 'FAILURE'), 1)
  705. self.assertEqual(self.countJobResults(builds, None), 1)
  706. builds = items[2].current_build_set.getBuilds()
  707. self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
  708. self.assertEqual(self.countJobResults(builds, None), 2)
  709. self.executor_server.hold_jobs_in_build = False
  710. self.executor_server.release()
  711. self.waitUntilSettled()
  712. self.assertEqual(len(self.builds), 0)
  713. self.assertEqual(len(self.history), 12)
  714. self.assertEqual(A.data['status'], 'MERGED')
  715. self.assertEqual(B.data['status'], 'NEW')
  716. self.assertEqual(C.data['status'], 'MERGED')
  717. self.assertEqual(A.reported, 2)
  718. self.assertEqual(B.reported, 2)
  719. self.assertEqual(C.reported, 2)
  720. def test_failed_change_at_head_with_queue(self):
  721. "Test that if a change at the head fails, queued jobs are canceled"
  722. self.gearman_server.hold_jobs_in_queue = True
  723. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  724. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  725. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  726. A.addApproval('Code-Review', 2)
  727. B.addApproval('Code-Review', 2)
  728. C.addApproval('Code-Review', 2)
  729. self.executor_server.failJob('project-test1', A)
  730. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  731. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  732. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  733. self.waitUntilSettled()
  734. queue = self.gearman_server.getQueue()
  735. self.assertEqual(len(self.builds), 0)
  736. self.assertEqual(len(queue), 1)
  737. self.assertEqual(queue[0].name, b'executor:execute')
  738. job_args = json.loads(queue[0].arguments.decode('utf8'))
  739. self.assertEqual(job_args['job'], 'project-merge')
  740. self.assertEqual(job_args['items'][0]['number'], '%d' % A.number)
  741. self.gearman_server.release('.*-merge')
  742. self.waitUntilSettled()
  743. self.gearman_server.release('.*-merge')
  744. self.waitUntilSettled()
  745. self.gearman_server.release('.*-merge')
  746. self.waitUntilSettled()
  747. queue = self.gearman_server.getQueue()
  748. self.assertEqual(len(self.builds), 0)
  749. self.assertEqual(len(queue), 6)
  750. self.assertEqual(
  751. json.loads(queue[0].arguments.decode('utf8'))['job'],
  752. 'project-test1')
  753. self.assertEqual(
  754. json.loads(queue[1].arguments.decode('utf8'))['job'],
  755. 'project-test2')
  756. self.assertEqual(
  757. json.loads(queue[2].arguments.decode('utf8'))['job'],
  758. 'project-test1')
  759. self.assertEqual(
  760. json.loads(queue[3].arguments.decode('utf8'))['job'],
  761. 'project-test2')
  762. self.assertEqual(
  763. json.loads(queue[4].arguments.decode('utf8'))['job'],
  764. 'project-test1')
  765. self.assertEqual(
  766. json.loads(queue[5].arguments.decode('utf8'))['job'],
  767. 'project-test2')
  768. self.release(queue[0])
  769. self.waitUntilSettled()
  770. self.assertEqual(len(self.builds), 0)
  771. queue = self.gearman_server.getQueue()
  772. self.assertEqual(len(queue), 2) # project-test2, project-merge for B
  773. self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 0)
  774. self.gearman_server.hold_jobs_in_queue = False
  775. self.gearman_server.release()
  776. self.waitUntilSettled()
  777. self.assertEqual(len(self.builds), 0)
  778. self.assertEqual(len(self.history), 11)
  779. self.assertEqual(A.data['status'], 'NEW')
  780. self.assertEqual(B.data['status'], 'MERGED')
  781. self.assertEqual(C.data['status'], 'MERGED')
  782. self.assertEqual(A.reported, 2)
  783. self.assertEqual(B.reported, 2)
  784. self.assertEqual(C.reported, 2)
  785. def _test_time_database(self, iteration):
  786. self.executor_server.hold_jobs_in_build = True
  787. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  788. A.addApproval('Code-Review', 2)
  789. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  790. self.waitUntilSettled()
  791. time.sleep(2)
  792. data = json.loads(self.scheds.first.sched
  793. .formatStatusJSON('tenant-one'))
  794. found_job = None
  795. for pipeline in data['pipelines']:
  796. if pipeline['name'] != 'gate':
  797. continue
  798. for queue in pipeline['change_queues']:
  799. for head in queue['heads']:
  800. for item in head:
  801. for job in item['jobs']:
  802. if job['name'] == 'project-merge':
  803. found_job = job
  804. break
  805. self.assertIsNotNone(found_job)
  806. if iteration == 1:
  807. self.assertIsNotNone(found_job['estimated_time'])
  808. self.assertIsNone(found_job['remaining_time'])
  809. else:
  810. self.assertIsNotNone(found_job['estimated_time'])
  811. self.assertTrue(found_job['estimated_time'] >= 2)
  812. self.assertIsNotNone(found_job['remaining_time'])
  813. self.executor_server.hold_jobs_in_build = False
  814. self.executor_server.release()
  815. self.waitUntilSettled()
  816. def test_time_database(self):
  817. "Test the time database"
  818. self._test_time_database(1)
  819. self._test_time_database(2)
  820. def test_two_failed_changes_at_head(self):
  821. "Test that changes are reparented correctly if 2 fail at head"
  822. self.executor_server.hold_jobs_in_build = True
  823. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  824. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  825. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  826. A.addApproval('Code-Review', 2)
  827. B.addApproval('Code-Review', 2)
  828. C.addApproval('Code-Review', 2)
  829. self.executor_server.failJob('project-test1', A)
  830. self.executor_server.failJob('project-test1', B)
  831. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  832. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  833. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  834. self.waitUntilSettled()
  835. self.executor_server.release('.*-merge')
  836. self.waitUntilSettled()
  837. self.executor_server.release('.*-merge')
  838. self.waitUntilSettled()
  839. self.executor_server.release('.*-merge')
  840. self.waitUntilSettled()
  841. self.assertEqual(len(self.builds), 6)
  842. self.assertEqual(self.builds[0].name, 'project-test1')
  843. self.assertEqual(self.builds[1].name, 'project-test2')
  844. self.assertEqual(self.builds[2].name, 'project-test1')
  845. self.assertEqual(self.builds[3].name, 'project-test2')
  846. self.assertEqual(self.builds[4].name, 'project-test1')
  847. self.assertEqual(self.builds[5].name, 'project-test2')
  848. self.assertTrue(self.builds[0].hasChanges(A))
  849. self.assertTrue(self.builds[2].hasChanges(A))
  850. self.assertTrue(self.builds[2].hasChanges(B))
  851. self.assertTrue(self.builds[4].hasChanges(A))
  852. self.assertTrue(self.builds[4].hasChanges(B))
  853. self.assertTrue(self.builds[4].hasChanges(C))
  854. # Fail change B first
  855. self.release(self.builds[2])
  856. self.waitUntilSettled()
  857. # restart of C after B failure
  858. self.executor_server.release('.*-merge')
  859. self.waitUntilSettled()
  860. self.assertEqual(len(self.builds), 5)
  861. self.assertEqual(self.builds[0].name, 'project-test1')
  862. self.assertEqual(self.builds[1].name, 'project-test2')
  863. self.assertEqual(self.builds[2].name, 'project-test2')
  864. self.assertEqual(self.builds[3].name, 'project-test1')
  865. self.assertEqual(self.builds[4].name, 'project-test2')
  866. self.assertTrue(self.builds[1].hasChanges(A))
  867. self.assertTrue(self.builds[2].hasChanges(A))
  868. self.assertTrue(self.builds[2].hasChanges(B))
  869. self.assertTrue(self.builds[4].hasChanges(A))
  870. self.assertFalse(self.builds[4].hasChanges(B))
  871. self.assertTrue(self.builds[4].hasChanges(C))
  872. # Finish running all passing jobs for change A
  873. self.release(self.builds[1])
  874. self.waitUntilSettled()
  875. # Fail and report change A
  876. self.release(self.builds[0])
  877. self.waitUntilSettled()
  878. # restart of B,C after A failure
  879. self.executor_server.release('.*-merge')
  880. self.waitUntilSettled()
  881. self.executor_server.release('.*-merge')
  882. self.waitUntilSettled()
  883. self.assertEqual(len(self.builds), 4)
  884. self.assertEqual(self.builds[0].name, 'project-test1') # B
  885. self.assertEqual(self.builds[1].name, 'project-test2') # B
  886. self.assertEqual(self.builds[2].name, 'project-test1') # C
  887. self.assertEqual(self.builds[3].name, 'project-test2') # C
  888. self.assertFalse(self.builds[1].hasChanges(A))
  889. self.assertTrue(self.builds[1].hasChanges(B))
  890. self.assertFalse(self.builds[1].hasChanges(C))
  891. self.assertFalse(self.builds[2].hasChanges(A))
  892. # After A failed and B and C restarted, B should be back in
  893. # C's tests because it has not failed yet.
  894. self.assertTrue(self.builds[2].hasChanges(B))
  895. self.assertTrue(self.builds[2].hasChanges(C))
  896. self.executor_server.hold_jobs_in_build = False
  897. self.executor_server.release()
  898. self.waitUntilSettled()
  899. self.assertEqual(len(self.builds), 0)
  900. self.assertEqual(len(self.history), 21)
  901. self.assertEqual(A.data['status'], 'NEW')
  902. self.assertEqual(B.data['status'], 'NEW')
  903. self.assertEqual(C.data['status'], 'MERGED')
  904. self.assertEqual(A.reported, 2)
  905. self.assertEqual(B.reported, 2)
  906. self.assertEqual(C.reported, 2)
  907. def test_patch_order(self):
  908. "Test that dependent patches are tested in the right order"
  909. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  910. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  911. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  912. A.addApproval('Code-Review', 2)
  913. B.addApproval('Code-Review', 2)
  914. C.addApproval('Code-Review', 2)
  915. M2 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M2')
  916. M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
  917. M2.setMerged()
  918. M1.setMerged()
  919. # C -> B -> A -> M1 -> M2
  920. # M2 is here to make sure it is never queried. If it is, it
  921. # means zuul is walking down the entire history of merged
  922. # changes.
  923. C.setDependsOn(B, 1)
  924. B.setDependsOn(A, 1)
  925. A.setDependsOn(M1, 1)
  926. M1.setDependsOn(M2, 1)
  927. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  928. self.waitUntilSettled()
  929. self.assertEqual(A.data['status'], 'NEW')
  930. self.assertEqual(B.data['status'], 'NEW')
  931. self.assertEqual(C.data['status'], 'NEW')
  932. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  933. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  934. self.waitUntilSettled()
  935. self.assertEqual(M2.queried, 0)
  936. self.assertEqual(A.data['status'], 'MERGED')
  937. self.assertEqual(B.data['status'], 'MERGED')
  938. self.assertEqual(C.data['status'], 'MERGED')
  939. self.assertEqual(A.reported, 2)
  940. self.assertEqual(B.reported, 2)
  941. self.assertEqual(C.reported, 2)
  942. def test_needed_changes_enqueue(self):
  943. "Test that a needed change is enqueued ahead"
  944. # A Given a git tree like this, if we enqueue
  945. # / \ change C, we should walk up and down the tree
  946. # B G and enqueue changes in the order ABCDEFG.
  947. # /|\ This is also the order that you would get if
  948. # *C E F you enqueued changes in the order ABCDEFG, so
  949. # / the ordering is stable across re-enqueue events.
  950. # D
  951. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  952. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  953. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  954. D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
  955. E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
  956. F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')
  957. G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')
  958. B.setDependsOn(A, 1)
  959. C.setDependsOn(B, 1)
  960. D.setDependsOn(C, 1)
  961. E.setDependsOn(B, 1)
  962. F.setDependsOn(B, 1)
  963. G.setDependsOn(A, 1)
  964. A.addApproval('Code-Review', 2)
  965. B.addApproval('Code-Review', 2)
  966. C.addApproval('Code-Review', 2)
  967. D.addApproval('Code-Review', 2)
  968. E.addApproval('Code-Review', 2)
  969. F.addApproval('Code-Review', 2)
  970. G.addApproval('Code-Review', 2)
  971. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  972. self.waitUntilSettled()
  973. self.assertEqual(A.data['status'], 'NEW')
  974. self.assertEqual(B.data['status'], 'NEW')
  975. self.assertEqual(C.data['status'], 'NEW')
  976. self.assertEqual(D.data['status'], 'NEW')
  977. self.assertEqual(E.data['status'], 'NEW')
  978. self.assertEqual(F.data['status'], 'NEW')
  979. self.assertEqual(G.data['status'], 'NEW')
  980. # We're about to add approvals to changes without adding the
  981. # triggering events to Zuul, so that we can be sure that it is
  982. # enqueing the changes based on dependencies, not because of
  983. # triggering events. Since it will have the changes cached
  984. # already (without approvals), we need to clear the cache
  985. # first.
  986. for connection in self.scheds.first.connections.connections.values():
  987. connection.maintainCache([])
  988. self.executor_server.hold_jobs_in_build = True
  989. A.addApproval('Approved', 1)
  990. B.addApproval('Approved', 1)
  991. D.addApproval('Approved', 1)
  992. E.addApproval('Approved', 1)
  993. F.addApproval('Approved', 1)
  994. G.addApproval('Approved', 1)
  995. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  996. for x in range(8):
  997. self.executor_server.release('.*-merge')
  998. self.waitUntilSettled()
  999. self.executor_server.hold_jobs_in_build = False
  1000. self.executor_server.release()
  1001. self.waitUntilSettled()
  1002. self.assertEqual(A.data['status'], 'MERGED')
  1003. self.assertEqual(B.data['status'], 'MERGED')
  1004. self.assertEqual(C.data['status'], 'MERGED')
  1005. self.assertEqual(D.data['status'], 'MERGED')
  1006. self.assertEqual(E.data['status'], 'MERGED')
  1007. self.assertEqual(F.data['status'], 'MERGED')
  1008. self.assertEqual(G.data['status'], 'MERGED')
  1009. self.assertEqual(A.reported, 2)
  1010. self.assertEqual(B.reported, 2)
  1011. self.assertEqual(C.reported, 2)
  1012. self.assertEqual(D.reported, 2)
  1013. self.assertEqual(E.reported, 2)
  1014. self.assertEqual(F.reported, 2)
  1015. self.assertEqual(G.reported, 2)
  1016. self.assertEqual(self.history[6].changes,
  1017. '1,1 2,1 3,1 4,1 5,1 6,1 7,1')
  1018. def test_source_cache(self):
  1019. "Test that the source cache operates correctly"
  1020. self.executor_server.hold_jobs_in_build = True
  1021. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1022. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  1023. X = self.fake_gerrit.addFakeChange('org/project', 'master', 'X')
  1024. A.addApproval('Code-Review', 2)
  1025. B.addApproval('Code-Review', 2)
  1026. M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
  1027. M1.setMerged()
  1028. B.setDependsOn(A, 1)
  1029. A.setDependsOn(M1, 1)
  1030. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  1031. self.fake_gerrit.addEvent(X.getPatchsetCreatedEvent(1))
  1032. self.waitUntilSettled()
  1033. for build in self.builds:
  1034. if build.pipeline == 'check':
  1035. build.release()
  1036. self.waitUntilSettled()
  1037. for build in self.builds:
  1038. if build.pipeline == 'check':
  1039. build.release()
  1040. self.waitUntilSettled()
  1041. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  1042. self.waitUntilSettled()
  1043. self.log.debug("len %s" % self.fake_gerrit._change_cache.keys())
  1044. # there should still be changes in the cache
  1045. self.assertNotEqual(len(self.fake_gerrit._change_cache.keys()), 0)
  1046. self.executor_server.hold_jobs_in_build = False
  1047. self.executor_server.release()
  1048. self.waitUntilSettled()
  1049. self.assertEqual(A.data['status'], 'MERGED')
  1050. self.assertEqual(B.data['status'], 'MERGED')
  1051. self.assertEqual(A.queried, 2) # Initial and isMerged
  1052. self.assertEqual(B.queried, 3) # Initial A, refresh from B, isMerged
  1053. def test_can_merge(self):
  1054. "Test whether a change is ready to merge"
  1055. # TODO: move to test_gerrit (this is a unit test!)
  1056. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1057. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  1058. (trusted, project) = tenant.getProject('org/project')
  1059. source = project.source
  1060. # TODO(pabelanger): As we add more source / trigger APIs we should make
  1061. # it easier for users to create events for testing.
  1062. event = zuul.model.TriggerEvent()
  1063. event.trigger_name = 'gerrit'
  1064. event.change_number = '1'
  1065. event.patch_number = '2'
  1066. a = source.getChange(event)
  1067. mgr = tenant.layout.pipelines['gate'].manager
  1068. self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))
  1069. A.addApproval('Code-Review', 2)
  1070. a = source.getChange(event, refresh=True)
  1071. self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))
  1072. A.addApproval('Approved', 1)
  1073. a = source.getChange(event, refresh=True)
  1074. self.assertTrue(source.canMerge(a, mgr.getSubmitAllowNeeds()))
  1075. def test_project_merge_conflict(self):
  1076. "Test that gate merge conflicts are handled properly"
  1077. self.gearman_server.hold_jobs_in_queue = True
  1078. A = self.fake_gerrit.addFakeChange('org/project',
  1079. 'master', 'A',
  1080. files={'conflict': 'foo'})
  1081. B = self.fake_gerrit.addFakeChange('org/project',
  1082. 'master', 'B',
  1083. files={'conflict': 'bar'})
  1084. C = self.fake_gerrit.addFakeChange('org/project',
  1085. 'master', 'C')
  1086. A.addApproval('Code-Review', 2)
  1087. B.addApproval('Code-Review', 2)
  1088. C.addApproval('Code-Review', 2)
  1089. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  1090. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  1091. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  1092. self.waitUntilSettled()
  1093. self.assertEqual(A.reported, 1)
  1094. self.assertEqual(C.reported, 1)
  1095. self.gearman_server.release('project-merge')
  1096. self.waitUntilSettled()
  1097. self.gearman_server.release('project-merge')
  1098. self.waitUntilSettled()
  1099. self.gearman_server.release('project-merge')
  1100. self.waitUntilSettled()
  1101. self.gearman_server.hold_jobs_in_queue = False
  1102. self.gearman_server.release()
  1103. self.waitUntilSettled()
  1104. self.assertEqual(A.data['status'], 'MERGED')
  1105. self.assertEqual(B.data['status'], 'NEW')
  1106. self.assertEqual(C.data['status'], 'MERGED')
  1107. self.assertEqual(A.reported, 2)
  1108. self.assertIn('Merge Failed', B.messages[-1])
  1109. self.assertEqual(C.reported, 2)
  1110. self.assertHistory([
  1111. dict(name='project-merge', result='SUCCESS', changes='1,1'),
  1112. dict(name='project-test1', result='SUCCESS', changes='1,1'),
  1113. dict(name='project-test2', result='SUCCESS', changes='1,1'),
  1114. dict(name='project-merge', result='SUCCESS', changes='1,1 3,1'),
  1115. dict(name='project-test1', result='SUCCESS', changes='1,1 3,1'),
  1116. dict(name='project-test2', result='SUCCESS', changes='1,1 3,1'),
  1117. ], ordered=False)
  1118. def test_delayed_merge_conflict(self):
  1119. "Test that delayed check merge conflicts are handled properly"
  1120. # Hold jobs in the gearman queue so that we can test whether
  1121. # the executor sucesfully merges a change based on an old
  1122. # repo state (frozen by the scheduler) which would otherwise
  1123. # conflict.
  1124. self.gearman_server.hold_jobs_in_queue = True
  1125. A = self.fake_gerrit.addFakeChange('org/project',
  1126. 'master', 'A',
  1127. files={'conflict': 'foo'})
  1128. B = self.fake_gerrit.addFakeChange('org/project',
  1129. 'master', 'B',
  1130. files={'conflict': 'bar'})
  1131. C = self.fake_gerrit.addFakeChange('org/project',
  1132. 'master', 'C')
  1133. C.setDependsOn(B, 1)
  1134. # A enters the gate queue; B and C enter the check queue
  1135. A.addApproval('Code-Review', 2)
  1136. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  1137. self.waitUntilSettled()
  1138. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  1139. self.waitUntilSettled()
  1140. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  1141. self.waitUntilSettled()
  1142. self.assertEqual(A.reported, 1)
  1143. self.assertEqual(B.reported, 0) # Check does not report start
  1144. self.assertEqual(C.reported, 0) # Check does not report start
  1145. # A merges while B and C are queued in check
  1146. # Release A project-merge
  1147. queue = self.gearman_server.getQueue()
  1148. self.release(queue[0])
  1149. self.waitUntilSettled()
  1150. # Release A project-test*
  1151. # gate has higher precedence, so A's test jobs are added in
  1152. # front of the merge jobs for B and C
  1153. queue = self.gearman_server.getQueue()
  1154. self.release(queue[0])
  1155. self.release(queue[1])
  1156. self.waitUntilSettled()
  1157. self.assertEqual(A.data['status'], 'MERGED')
  1158. self.assertEqual(B.data['status'], 'NEW')
  1159. self.assertEqual(C.data['status'], 'NEW')
  1160. self.assertEqual(A.reported, 2)
  1161. self.assertEqual(B.reported, 0)
  1162. self.assertEqual(C.reported, 0)
  1163. self.assertHistory([
  1164. dict(name='project-merge', result='SUCCESS', changes='1,1'),
  1165. dict(name='project-test1', result='SUCCESS', changes='1,1'),
  1166. dict(name='project-test2', result='SUCCESS', changes='1,1'),
  1167. ], ordered=False)
  1168. # B and C report merge conflicts
  1169. # Release B project-merge
  1170. queue = self.gearman_server.getQueue()
  1171. self.release(queue[0])
  1172. self.waitUntilSettled()
  1173. # Release C
  1174. self.gearman_server.hold_jobs_in_queue = False
  1175. self.gearman_server.release()
  1176. self.waitUntilSettled()
  1177. self.assertEqual(A.data['status'], 'MERGED')
  1178. self.assertEqual(B.data['status'], 'NEW')
  1179. self.assertEqual(C.data['status'], 'NEW')
  1180. self.assertEqual(A.reported, 2)
  1181. self.assertEqual(B.reported, 1)
  1182. self.assertEqual(C.reported, 1)
  1183. self.assertHistory([
  1184. dict(name='project-merge', result='SUCCESS', changes='1,1'),
  1185. dict(name='project-test1', result='SUCCESS', changes='1,1'),
  1186. dict(name='project-test2', result='SUCCESS', changes='1,1'),
  1187. dict(name='project-merge', result='SUCCESS', changes='2,1'),
  1188. dict(name='project-test1', result='SUCCESS', changes='2,1'),
  1189. dict(name='project-test2', result='SUCCESS', changes='2,1'),
  1190. dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'),
  1191. dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'),
  1192. dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'),
  1193. ], ordered=False)
  1194. def test_post(self):
  1195. "Test that post jobs run"
  1196. p = "review.example.com/org/project"
  1197. upstream = self.getUpstreamRepos([p])
  1198. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1199. A.setMerged()
  1200. A_commit = str(upstream[p].commit('master'))
  1201. self.log.debug("A commit: %s" % A_commit)
  1202. e = {
  1203. "type": "ref-updated",
  1204. "submitter": {
  1205. "name": "User Name",
  1206. },
  1207. "refUpdate": {
  1208. "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
  1209. "newRev": A_commit,
  1210. "refName": "master",
  1211. "project": "org/project",
  1212. }
  1213. }
  1214. self.fake_gerrit.addEvent(e)
  1215. self.waitUntilSettled()
  1216. job_names = [x.name for x in self.history]
  1217. self.assertEqual(len(self.history), 1)
  1218. self.assertIn('project-post', job_names)
  1219. def test_post_ignore_deletes(self):
  1220. "Test that deleting refs does not trigger post jobs"
  1221. e = {
  1222. "type": "ref-updated",
  1223. "submitter": {
  1224. "name": "User Name",
  1225. },
  1226. "refUpdate": {
  1227. "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
  1228. "newRev": "0000000000000000000000000000000000000000",
  1229. "refName": "master",
  1230. "project": "org/project",
  1231. }
  1232. }
  1233. self.fake_gerrit.addEvent(e)
  1234. self.waitUntilSettled()
  1235. job_names = [x.name for x in self.history]
  1236. self.assertEqual(len(self.history), 0)
  1237. self.assertNotIn('project-post', job_names)
  1238. @simple_layout('layouts/dont-ignore-ref-deletes.yaml')
  1239. def test_post_ignore_deletes_negative(self):
  1240. "Test that deleting refs does trigger post jobs"
  1241. e = {
  1242. "type": "ref-updated",
  1243. "submitter": {
  1244. "name": "User Name",
  1245. },
  1246. "refUpdate": {
  1247. "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
  1248. "newRev": "0000000000000000000000000000000000000000",
  1249. "refName": "testbranch",
  1250. "project": "org/project",
  1251. }
  1252. }
  1253. self.fake_gerrit.addEvent(e)
  1254. self.waitUntilSettled()
  1255. job_names = [x.name for x in self.history]
  1256. self.assertEqual(len(self.history), 1)
  1257. self.assertIn('project-post', job_names)
  1258. @skip("Disabled for early v3 development")
  1259. def test_build_configuration_branch_interaction(self):
  1260. "Test that switching between branches works"
  1261. self.test_build_configuration()
  1262. self.test_build_configuration_branch()
  1263. # C has been merged, undo that
  1264. path = os.path.join(self.upstream_root, "org/project")
  1265. repo = git.Repo(path)
  1266. repo.heads.master.commit = repo.commit('init')
  1267. self.test_build_configuration()
  1268. def test_dependent_changes_rebase(self):
  1269. # Test that no errors occur when we walk a dependency tree
  1270. # with an unused leaf node due to a rebase.
  1271. # Start by constructing: C -> B -> A
  1272. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1273. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  1274. B.setDependsOn(A, 1)
  1275. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  1276. C.setDependsOn(B, 1)
  1277. # Then rebase to form: D -> C -> A
  1278. C.addPatchset() # C,2
  1279. C.setDependsOn(A, 1)
  1280. D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
  1281. D.setDependsOn(C, 2)
  1282. # Walk the entire tree
  1283. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  1284. self.waitUntilSettled()
  1285. self.assertEqual(len(self.history), 3)
  1286. # Verify that walking just part of the tree still works
  1287. self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
  1288. self.waitUntilSettled()
  1289. self.assertEqual(len(self.history), 6)
  1290. def test_dependent_changes_dequeue(self):
  1291. "Test that dependent patches are not needlessly tested"
  1292. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1293. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  1294. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  1295. A.addApproval('Code-Review', 2)
  1296. B.addApproval('Code-Review', 2)
  1297. C.addApproval('Code-Review', 2)
  1298. M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
  1299. M1.setMerged()
  1300. # C -> B -> A -> M1
  1301. C.setDependsOn(B, 1)
  1302. B.setDependsOn(A, 1)
  1303. A.setDependsOn(M1, 1)
  1304. self.executor_server.failJob('project-merge', A)
  1305. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  1306. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  1307. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  1308. self.waitUntilSettled()
  1309. self.assertEqual(A.data['status'], 'NEW')
  1310. self.assertEqual(A.reported, 2)
  1311. self.assertEqual(B.data['status'], 'NEW')
  1312. self.assertEqual(B.reported, 2)
  1313. self.assertEqual(C.data['status'], 'NEW')
  1314. self.assertIn('This change depends on a change that failed to merge.',
  1315. C.messages[-1])
  1316. self.assertEqual(len(self.history), 1)
  1317. def test_failing_dependent_changes(self):
  1318. "Test that failing dependent patches are taken out of stream"
  1319. self.executor_server.hold_jobs_in_build = True
  1320. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1321. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  1322. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  1323. D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
  1324. E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
  1325. A.addApproval('Code-Review', 2)
  1326. B.addApproval('Code-Review', 2)
  1327. C.addApproval('Code-Review', 2)
  1328. D.addApproval('Code-Review', 2)
  1329. E.addApproval('Code-Review', 2)
  1330. # E, D -> C -> B, A
  1331. D.setDependsOn(C, 1)
  1332. C.setDependsOn(B, 1)
  1333. self.executor_server.failJob('project-test1', B)
  1334. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  1335. self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
  1336. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  1337. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  1338. self.fake_gerrit.addEvent(E.addApproval('Approved', 1))
  1339. self.waitUntilSettled()
  1340. self.executor_server.release('.*-merge')
  1341. self.waitUntilSettled()
  1342. self.executor_server.release('.*-merge')
  1343. self.waitUntilSettled()
  1344. self.executor_server.release('.*-merge')
  1345. self.waitUntilSettled()
  1346. self.executor_server.release('.*-merge')
  1347. self.waitUntilSettled()
  1348. self.executor_server.release('.*-merge')
  1349. self.waitUntilSettled()
  1350. self.executor_server.hold_jobs_in_build = False
  1351. for build in self.builds:
  1352. if build.parameters['zuul']['change'] != '1':
  1353. build.release()
  1354. self.waitUntilSettled()
  1355. self.executor_server.release()
  1356. self.waitUntilSettled()
  1357. self.assertEqual(A.data['status'], 'MERGED')
  1358. self.assertEqual(A.reported, 2)
  1359. self.assertIn('Build succeeded', A.messages[1])
  1360. self.assertEqual(B.data['status'], 'NEW')
  1361. self.assertEqual(B.reported, 2)
  1362. self.assertIn('Build failed', B.messages[1])
  1363. self.assertEqual(C.data['status'], 'NEW')
  1364. self.assertEqual(C.reported, 2)
  1365. self.assertIn('depends on a change', C.messages[1])
  1366. self.assertEqual(D.data['status'], 'NEW')
  1367. self.assertEqual(D.reported, 2)
  1368. self.assertIn('depends on a change', D.messages[1])
  1369. self.assertEqual(E.data['status'], 'MERGED')
  1370. self.assertEqual(E.reported, 2)
  1371. self.assertIn('Build succeeded', E.messages[1])
  1372. self.assertEqual(len(self.history), 18)
  1373. def test_head_is_dequeued_once(self):
  1374. "Test that if a change at the head fails it is dequeued only once"
  1375. # If it's dequeued more than once, we should see extra
  1376. # aborted jobs.
  1377. self.executor_server.hold_jobs_in_build = True
  1378. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1379. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  1380. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  1381. A.addApproval('Code-Review', 2)
  1382. B.addApproval('Code-Review', 2)
  1383. C.addApproval('Code-Review', 2)
  1384. self.executor_server.failJob('project-test1', A)
  1385. self.executor_server.failJob('project-test2', A)
  1386. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  1387. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  1388. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  1389. self.waitUntilSettled()
  1390. self.assertEqual(len(self.builds), 1)
  1391. self.assertEqual(self.builds[0].name, 'project-merge')
  1392. self.assertTrue(self.builds[0].hasChanges(A))
  1393. self.executor_server.release('.*-merge')
  1394. self.waitUntilSettled()
  1395. self.executor_server.release('.*-merge')
  1396. self.waitUntilSettled()
  1397. self.executor_server.release('.*-merge')
  1398. self.waitUntilSettled()
  1399. self.assertEqual(len(self.builds), 6)
  1400. self.assertEqual(self.builds[0].name, 'project-test1')
  1401. self.assertEqual(self.builds[1].name, 'project-test2')
  1402. self.assertEqual(self.builds[2].name, 'project-test1')
  1403. self.assertEqual(self.builds[3].name, 'project-test2')
  1404. self.assertEqual(self.builds[4].name, 'project-test1')
  1405. self.assertEqual(self.builds[5].name, 'project-test2')
  1406. self.release(self.builds[0])
  1407. self.waitUntilSettled()
  1408. self.assertEqual(len(self.builds), 2) # test2, merge for B
  1409. self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)
  1410. self.executor_server.hold_jobs_in_build = False
  1411. self.executor_server.release()
  1412. self.waitUntilSettled()
  1413. self.assertEqual(len(self.builds), 0)
  1414. self.assertEqual(len(self.history), 15)
  1415. self.assertEqual(A.data['status'], 'NEW')
  1416. self.assertEqual(B.data['status'], 'MERGED')
  1417. self.assertEqual(C.data['status'], 'MERGED')
  1418. self.assertEqual(A.reported, 2)
  1419. self.assertEqual(B.reported, 2)
  1420. self.assertEqual(C.reported, 2)
  1421. @simple_layout('layouts/nonvoting-job-approval.yaml')
  1422. def test_nonvoting_job_approval(self):
  1423. "Test that non-voting jobs don't vote but leave approval"
  1424. A = self.fake_gerrit.addFakeChange('org/nonvoting-project',
  1425. 'master', 'A')
  1426. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  1427. self.executor_server.failJob('nonvoting-project-test2', A)
  1428. self.waitUntilSettled()
  1429. self.assertEqual(A.data['status'], 'NEW')
  1430. self.assertEqual(A.reported, 1)
  1431. self.assertEqual(
  1432. self.getJobFromHistory('nonvoting-project-test1').result,
  1433. 'SUCCESS')
  1434. self.assertEqual(
  1435. self.getJobFromHistory('nonvoting-project-test2').result,
  1436. 'FAILURE')
  1437. self.assertFalse(self.getJobFromHistory('nonvoting-project-test1').
  1438. parameters['zuul']['voting'])
  1439. self.assertFalse(self.getJobFromHistory('nonvoting-project-test2').
  1440. parameters['zuul']['voting'])
  1441. self.assertEqual(A.patchsets[0]['approvals'][0]['value'], "1")
  1442. @simple_layout('layouts/nonvoting-job.yaml')
  1443. def test_nonvoting_job(self):
  1444. "Test that non-voting jobs don't vote."
  1445. A = self.fake_gerrit.addFakeChange('org/nonvoting-project',
  1446. 'master', 'A')
  1447. A.addApproval('Code-Review', 2)
  1448. self.executor_server.failJob('nonvoting-project-test2', A)
  1449. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  1450. self.waitUntilSettled()
  1451. self.assertEqual(A.data['status'], 'MERGED')
  1452. self.assertEqual(A.reported, 2)
  1453. self.assertEqual(
  1454. self.getJobFromHistory('nonvoting-project-merge').result,
  1455. 'SUCCESS')
  1456. self.assertEqual(
  1457. self.getJobFromHistory('nonvoting-project-test1').result,
  1458. 'SUCCESS')
  1459. self.assertEqual(
  1460. self.getJobFromHistory('nonvoting-project-test2').result,
  1461. 'FAILURE')
  1462. self.assertTrue(self.getJobFromHistory('nonvoting-project-merge').
  1463. parameters['zuul']['voting'])
  1464. self.assertTrue(self.getJobFromHistory('nonvoting-project-test1').
  1465. parameters['zuul']['voting'])
  1466. self.assertFalse(self.getJobFromHistory('nonvoting-project-test2').
  1467. parameters['zuul']['voting'])
  1468. def test_check_queue_success(self):
  1469. "Test successful check queue jobs."
  1470. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1471. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  1472. self.waitUntilSettled()
  1473. self.assertEqual(A.data['status'], 'NEW')
  1474. self.assertEqual(A.reported, 1)
  1475. self.assertEqual(self.getJobFromHistory('project-merge').result,
  1476. 'SUCCESS')
  1477. self.assertEqual(self.getJobFromHistory('project-test1').result,
  1478. 'SUCCESS')
  1479. self.assertEqual(self.getJobFromHistory('project-test2').result,
  1480. 'SUCCESS')
  1481. def test_check_queue_failure(self):
  1482. "Test failed check queue jobs."
  1483. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1484. self.executor_server.failJob('project-test2', A)
  1485. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  1486. self.waitUntilSettled()
  1487. self.assertEqual(A.data['status'], 'NEW')
  1488. self.assertEqual(A.reported, 1)
  1489. self.assertEqual(self.getJobFromHistory('project-merge').result,
  1490. 'SUCCESS')
  1491. self.assertEqual(self.getJobFromHistory('project-test1').result,
  1492. 'SUCCESS')
  1493. self.assertEqual(self.getJobFromHistory('project-test2').result,
  1494. 'FAILURE')
  1495. @simple_layout('layouts/autohold.yaml')
  1496. def test_autohold(self):
  1497. client = zuul.rpcclient.RPCClient('127.0.0.1',
  1498. self.gearman_server.port)
  1499. self.addCleanup(client.shutdown)
  1500. r = client.autohold('tenant-one', 'org/project', 'project-test2',
  1501. "", "", "reason text", 1)
  1502. self.assertTrue(r)
  1503. # There should be a record in ZooKeeper
  1504. request_list = self.scheds.first.sched.zk.getHoldRequests()
  1505. self.assertEqual(1, len(request_list))
  1506. request = self.scheds.first.sched.zk.getHoldRequest(request_list[0])
  1507. self.assertIsNotNone(request)
  1508. self.assertEqual('tenant-one', request.tenant)
  1509. self.assertEqual('review.example.com/org/project', request.project)
  1510. self.assertEqual('project-test2', request.job)
  1511. self.assertEqual('reason text', request.reason)
  1512. self.assertEqual(1, request.max_count)
  1513. self.assertEqual(0, request.current_count)
  1514. self.assertEqual([], request.nodes)
  1515. # First check that successful jobs do not autohold
  1516. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1517. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  1518. self.waitUntilSettled()
  1519. self.assertEqual(A.data['status'], 'NEW')
  1520. self.assertEqual(A.reported, 1)
  1521. # project-test2
  1522. self.assertEqual(self.history[0].result, 'SUCCESS')
  1523. # Check nodepool for a held node
  1524. held_node = None
  1525. for node in self.fake_nodepool.getNodes():
  1526. if node['state'] == zuul.model.STATE_HOLD:
  1527. held_node = node
  1528. break
  1529. self.assertIsNone(held_node)
  1530. # Now test that failed jobs are autoheld
  1531. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  1532. self.executor_server.failJob('project-test2', B)
  1533. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  1534. self.waitUntilSettled()
  1535. self.assertEqual(B.data['status'], 'NEW')
  1536. self.assertEqual(B.reported, 1)
  1537. # project-test2
  1538. self.assertEqual(self.history[1].result, 'FAILURE')
  1539. # Check nodepool for a held node
  1540. held_node = None
  1541. for node in self.fake_nodepool.getNodes():
  1542. if node['state'] == zuul.model.STATE_HOLD:
  1543. held_node = node
  1544. break
  1545. self.assertIsNotNone(held_node)
  1546. # Validate node has recorded the failed job
  1547. self.assertEqual(
  1548. held_node['hold_job'],
  1549. " ".join(['tenant-one',
  1550. 'review.example.com/org/project',
  1551. 'project-test2', '.*'])
  1552. )
  1553. self.assertEqual(held_node['comment'], "reason text")
  1554. # The hold request current_count should have incremented
  1555. # and we should have recorded the held node ID.
  1556. request2 = self.scheds.first.sched.zk.getHoldRequest(request.id)
  1557. self.assertEqual(request.current_count + 1, request2.current_count)
  1558. self.assertEqual(1, len(request2.nodes))
  1559. self.assertEqual(1, len(request2.nodes[0]["nodes"]))
  1560. # Another failed change should not hold any more nodes
  1561. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  1562. self.executor_server.failJob('project-test2', C)
  1563. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  1564. self.waitUntilSettled()
  1565. self.assertEqual(C.data['status'], 'NEW')
  1566. self.assertEqual(C.reported, 1)
  1567. # project-test2
  1568. self.assertEqual(self.history[2].result, 'FAILURE')
  1569. held_nodes = 0
  1570. for node in self.fake_nodepool.getNodes():
  1571. if node['state'] == zuul.model.STATE_HOLD:
  1572. held_nodes += 1
  1573. self.assertEqual(held_nodes, 1)
  1574. # request current_count should not have changed
  1575. request3 = self.scheds.first.sched.zk.getHoldRequest(request2.id)
  1576. self.assertEqual(request2.current_count, request3.current_count)
  1577. # Deleting hold request should set held nodes to used
  1578. self.scheds.first.sched.zk.deleteHoldRequest(request3)
  1579. node_states = [n['state'] for n in self.fake_nodepool.getNodes()]
  1580. self.assertEqual(3, len(node_states))
  1581. self.assertEqual([zuul.model.STATE_USED] * 3, node_states)
  1582. @simple_layout('layouts/autohold.yaml')
  1583. def test_autohold_info(self):
  1584. client = zuul.rpcclient.RPCClient('127.0.0.1',
  1585. self.gearman_server.port)
  1586. self.addCleanup(client.shutdown)
  1587. # Empty dict should be returned for "not found"
  1588. request = client.autohold_info("XxXxX")
  1589. self.assertEqual({}, request)
  1590. r = client.autohold('tenant-one', 'org/project', 'project-test2',
  1591. "", "", "reason text", 1)
  1592. self.assertTrue(r)
  1593. # There should be a record in ZooKeeper
  1594. request_list = self.scheds.first.sched.zk.getHoldRequests()
  1595. self.assertEqual(1, len(request_list))
  1596. request = self.scheds.first.sched.zk.getHoldRequest(request_list[0])
  1597. self.assertIsNotNone(request)
  1598. request = client.autohold_info(request.id)
  1599. self.assertNotEqual({}, request)
  1600. self.assertEqual('tenant-one', request['tenant'])
  1601. self.assertEqual('review.example.com/org/project', request['project'])
  1602. self.assertEqual('project-test2', request['job'])
  1603. self.assertEqual('reason text', request['reason'])
  1604. self.assertEqual(1, request['max_count'])
  1605. self.assertEqual(0, request['current_count'])
  1606. @simple_layout('layouts/autohold.yaml')
  1607. def test_autohold_delete(self):
  1608. client = zuul.rpcclient.RPCClient('127.0.0.1',
  1609. self.gearman_server.port)
  1610. self.addCleanup(client.shutdown)
  1611. r = client.autohold('tenant-one', 'org/project', 'project-test2',
  1612. "", "", "reason text", 1)
  1613. self.assertTrue(r)
  1614. # There should be a record in ZooKeeper
  1615. request_list = self.scheds.first.sched.zk.getHoldRequests()
  1616. self.assertEqual(1, len(request_list))
  1617. request = self.scheds.first.sched.zk.getHoldRequest(request_list[0])
  1618. self.assertIsNotNone(request)
  1619. # Delete and verify no more requests
  1620. self.assertTrue(client.autohold_delete(request.id))
  1621. request_list = self.scheds.first.sched.zk.getHoldRequests()
  1622. self.assertEqual([], request_list)
  1623. def _test_autohold_scoped(self, change_obj, change, ref):
  1624. client = zuul.rpcclient.RPCClient('127.0.0.1',
  1625. self.gearman_server.port)
  1626. self.addCleanup(client.shutdown)
  1627. # create two changes on the same project, and autohold request
  1628. # for one of them.
  1629. other = self.fake_gerrit.addFakeChange(
  1630. 'org/project', 'master', 'other'
  1631. )
  1632. r = client.autohold('tenant-one', 'org/project', 'project-test2',
  1633. str(change), ref, "reason text", 1)
  1634. self.assertTrue(r)
  1635. # First, check that an unrelated job does not trigger autohold, even
  1636. # when it failed
  1637. self.executor_server.failJob('project-test2', other)
  1638. self.fake_gerrit.addEvent(other.getPatchsetCreatedEvent(1))
  1639. self.waitUntilSettled()
  1640. self.assertEqual(other.data['status'], 'NEW')
  1641. self.assertEqual(other.reported, 1)
  1642. # project-test2
  1643. self.assertEqual(self.history[0].result, 'FAILURE')
  1644. # Check nodepool for a held node
  1645. held_node = None
  1646. for node in self.fake_nodepool.getNodes():
  1647. if node['state'] == zuul.model.STATE_HOLD:
  1648. held_node = node
  1649. break
  1650. self.assertIsNone(held_node)
  1651. # And then verify that failed job for the defined change
  1652. # triggers the autohold
  1653. self.executor_server.failJob('project-test2', change_obj)
  1654. self.fake_gerrit.addEvent(change_obj.getPatchsetCreatedEvent(1))
  1655. self.waitUntilSettled()
  1656. self.assertEqual(change_obj.data['status'], 'NEW')
  1657. self.assertEqual(change_obj.reported, 1)
  1658. # project-test2
  1659. self.assertEqual(self.history[1].result, 'FAILURE')
  1660. # Check nodepool for a held node
  1661. held_node = None
  1662. for node in self.fake_nodepool.getNodes():
  1663. if node['state'] == zuul.model.STATE_HOLD:
  1664. held_node = node
  1665. break
  1666. self.assertIsNotNone(held_node)
  1667. # Validate node has recorded the failed job
  1668. if change != "":
  1669. ref = "refs/changes/%s/%s/.*" % (
  1670. str(change_obj.number)[-1:], str(change_obj.number)
  1671. )
  1672. self.assertEqual(
  1673. held_node['hold_job'],
  1674. " ".join(['tenant-one',
  1675. 'review.example.com/org/project',
  1676. 'project-test2', ref])
  1677. )
  1678. self.assertEqual(held_node['comment'], "reason text")
  1679. @simple_layout('layouts/autohold.yaml')
  1680. def test_autohold_change(self):
  1681. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1682. self._test_autohold_scoped(A, change=A.number, ref="")
  1683. @simple_layout('layouts/autohold.yaml')
  1684. def test_autohold_ref(self):
  1685. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1686. ref = A.data['currentPatchSet']['ref']
  1687. self._test_autohold_scoped(A, change="", ref=ref)
  1688. @simple_layout('layouts/autohold.yaml')
  1689. def test_autohold_scoping(self):
  1690. client = zuul.rpcclient.RPCClient('127.0.0.1',
  1691. self.gearman_server.port)
  1692. self.addCleanup(client.shutdown)
  1693. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1694. # create three autohold requests, scoped to job, change and
  1695. # a specific ref
  1696. change = str(A.number)
  1697. ref = A.data['currentPatchSet']['ref']
  1698. r1 = client.autohold('tenant-one', 'org/project', 'project-test2',
  1699. "", "", "reason text", 1)
  1700. self.assertTrue(r1)
  1701. r2 = client.autohold('tenant-one', 'org/project', 'project-test2',
  1702. change, "", "reason text", 1)
  1703. self.assertTrue(r2)
  1704. r3 = client.autohold('tenant-one', 'org/project', 'project-test2',
  1705. "", ref, "reason text", 1)
  1706. self.assertTrue(r3)
  1707. # Fail 3 jobs for the same change, and verify that the autohold
  1708. # requests are fullfilled in the expected order: from the most
  1709. # specific towards the most generic one.
  1710. def _fail_job_and_verify_autohold_request(change_obj, ref_filter):
  1711. self.executor_server.failJob('project-test2', change_obj)
  1712. self.fake_gerrit.addEvent(change_obj.getPatchsetCreatedEvent(1))
  1713. self.waitUntilSettled()
  1714. # Check nodepool for a held node
  1715. held_node = None
  1716. for node in self.fake_nodepool.getNodes():
  1717. if node['state'] == zuul.model.STATE_HOLD:
  1718. held_node = node
  1719. break
  1720. self.assertIsNotNone(held_node)
  1721. self.assertEqual(
  1722. held_node['hold_job'],
  1723. " ".join(['tenant-one',
  1724. 'review.example.com/org/project',
  1725. 'project-test2', ref_filter])
  1726. )
  1727. self.assertFalse(held_node['_lock'], "Node %s is locked" %
  1728. (node['_oid'],))
  1729. self.fake_nodepool.removeNode(held_node)
  1730. _fail_job_and_verify_autohold_request(A, ref)
  1731. ref = "refs/changes/%s/%s/.*" % (str(change)[-1:], str(change))
  1732. _fail_job_and_verify_autohold_request(A, ref)
  1733. _fail_job_and_verify_autohold_request(A, ".*")
  1734. @simple_layout('layouts/autohold.yaml')
  1735. def test_autohold_ignores_aborted_jobs(self):
  1736. client = zuul.rpcclient.RPCClient('127.0.0.1',
  1737. self.gearman_server.port)
  1738. self.addCleanup(client.shutdown)
  1739. r = client.autohold('tenant-one', 'org/project', 'project-test2',
  1740. "", "", "reason text", 1)
  1741. self.assertTrue(r)
  1742. self.executor_server.hold_jobs_in_build = True
  1743. # Create a change that will have its job aborted
  1744. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1745. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  1746. self.waitUntilSettled()
  1747. # Creating new patchset on change A will abort A,1's job because
  1748. # a new patchset arrived replacing A,1 with A,2.
  1749. A.addPatchset()
  1750. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
  1751. self.waitUntilSettled()
  1752. self.executor_server.hold_jobs_in_build = False
  1753. self.executor_server.release()
  1754. self.waitUntilSettled()
  1755. self.assertEqual(A.data['status'], 'NEW')
  1756. # Note only the successful job for A,2 will report as we don't
  1757. # report aborted builds for old patchsets.
  1758. self.assertEqual(A.reported, 1)
  1759. # A,1 project-test2
  1760. self.assertEqual(self.history[0].result, 'ABORTED')
  1761. # A,2 project-test2
  1762. self.assertEqual(self.history[1].result, 'SUCCESS')
  1763. # Check nodepool for a held node
  1764. held_node = None
  1765. for node in self.fake_nodepool.getNodes():
  1766. if node['state'] == zuul.model.STATE_HOLD:
  1767. held_node = node
  1768. break
  1769. self.assertIsNone(held_node)
  1770. @simple_layout('layouts/autohold.yaml')
  1771. def test_autohold_hold_expiration(self):
  1772. client = zuul.rpcclient.RPCClient('127.0.0.1',
  1773. self.gearman_server.port)
  1774. self.addCleanup(client.shutdown)
  1775. r = client.autohold('tenant-one', 'org/project', 'project-test2',
  1776. "", "", "reason text", 1, node_hold_expiration=30)
  1777. self.assertTrue(r)
  1778. # Hold a failed job
  1779. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  1780. self.executor_server.failJob('project-test2', B)
  1781. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  1782. self.waitUntilSettled()
  1783. self.assertEqual(B.data['status'], 'NEW')
  1784. self.assertEqual(B.reported, 1)
  1785. # project-test2
  1786. self.assertEqual(self.history[0].result, 'FAILURE')
  1787. # Check nodepool for a held node
  1788. held_node = None
  1789. for node in self.fake_nodepool.getNodes():
  1790. if node['state'] == zuul.model.STATE_HOLD:
  1791. held_node = node
  1792. break
  1793. self.assertIsNotNone(held_node)
  1794. # Validate node has hold_expiration property
  1795. self.assertEqual(int(held_node['hold_expiration']), 30)
  1796. @simple_layout('layouts/autohold.yaml')
  1797. def test_autohold_list(self):
  1798. client = zuul.rpcclient.RPCClient('127.0.0.1',
  1799. self.gearman_server.port)
  1800. self.addCleanup(client.shutdown)
  1801. r = client.autohold('tenant-one', 'org/project', 'project-test2',
  1802. "", "", "reason text", 1)
  1803. self.assertTrue(r)
  1804. autohold_requests = client.autohold_list()
  1805. self.assertNotEqual([], autohold_requests)
  1806. self.assertEqual(1, len(autohold_requests))
  1807. request = autohold_requests[0]
  1808. self.assertEqual('tenant-one', request['tenant'])
  1809. self.assertIn('org/project', request['project'])
  1810. self.assertEqual('project-test2', request['job'])
  1811. self.assertEqual(".*", request['ref_filter'])
  1812. self.assertEqual("reason text", request['reason'])
  1813. @simple_layout('layouts/autohold.yaml')
  1814. def test_autohold_request_expiration(self):
  1815. orig_exp = Scheduler.EXPIRED_HOLD_REQUEST_TTL
  1816. def reset_exp():
  1817. self.scheds.first.sched.EXPIRED_HOLD_REQUEST_TTL = orig_exp
  1818. self.addCleanup(reset_exp)
  1819. client = zuul.rpcclient.RPCClient('127.0.0.1',
  1820. self.gearman_server.port)
  1821. self.addCleanup(client.shutdown)
  1822. # Temporarily shorten the hold request expiration time
  1823. r = client.autohold('tenant-one', 'org/project', 'project-test2',
  1824. "", "", "reason text", 1, 1)
  1825. self.assertTrue(r)
  1826. autohold_requests = client.autohold_list()
  1827. self.assertEqual(1, len(autohold_requests))
  1828. req = autohold_requests[0]
  1829. self.assertIsNone(req['expired'])
  1830. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1831. self.executor_server.failJob('project-test2', A)
  1832. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  1833. self.waitUntilSettled()
  1834. autohold_requests = client.autohold_list()
  1835. self.assertEqual(1, len(autohold_requests))
  1836. req = autohold_requests[0]
  1837. self.assertIsNotNone(req['expired'])
  1838. # Temporarily shorten hold time so that the hold request can be
  1839. # auto-deleted (which is done on another test failure). And wait
  1840. # long enough for nodes to expire and request to delete.
  1841. self.scheds.first.sched.EXPIRED_HOLD_REQUEST_TTL = 1
  1842. time.sleep(3)
  1843. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  1844. self.executor_server.failJob('project-test2', B)
  1845. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  1846. self.waitUntilSettled()
  1847. for _ in iterate_timeout(10, 'hold request expiration'):
  1848. if len(client.autohold_list()) == 0:
  1849. break
  1850. @simple_layout('layouts/three-projects.yaml')
  1851. def test_dependent_behind_dequeue(self):
  1852. # This particular test does a large amount of merges and needs a little
  1853. # more time to complete
  1854. self.wait_timeout = 120
  1855. "test that dependent changes behind dequeued changes work"
  1856. # This complicated test is a reproduction of a real life bug
  1857. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  1858. self.executor_server.hold_jobs_in_build = True
  1859. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  1860. B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
  1861. C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
  1862. D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
  1863. E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E')
  1864. F = self.fake_gerrit.addFakeChange('org/project3', 'master', 'F')
  1865. D.setDependsOn(C, 1)
  1866. E.setDependsOn(D, 1)
  1867. A.addApproval('Code-Review', 2)
  1868. B.addApproval('Code-Review', 2)
  1869. C.addApproval('Code-Review', 2)
  1870. D.addApproval('Code-Review', 2)
  1871. E.addApproval('Code-Review', 2)
  1872. F.addApproval('Code-Review', 2)
  1873. A.fail_merge = True
  1874. # Change object re-use in the gerrit trigger is hidden if
  1875. # changes are added in quick succession; waiting makes it more
  1876. # like real life.
  1877. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  1878. self.waitUntilSettled()
  1879. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  1880. self.waitUntilSettled()
  1881. self.executor_server.release('.*-merge')
  1882. self.waitUntilSettled()
  1883. self.executor_server.release('.*-merge')
  1884. self.waitUntilSettled()
  1885. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  1886. self.waitUntilSettled()
  1887. self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
  1888. self.waitUntilSettled()
  1889. self.fake_gerrit.addEvent(E.addApproval('Approved', 1))
  1890. self.waitUntilSettled()
  1891. self.fake_gerrit.addEvent(F.addApproval('Approved', 1))
  1892. self.waitUntilSettled()
  1893. self.executor_server.release('.*-merge')
  1894. self.waitUntilSettled()
  1895. self.executor_server.release('.*-merge')
  1896. self.waitUntilSettled()
  1897. self.executor_server.release('.*-merge')
  1898. self.waitUntilSettled()
  1899. self.executor_server.release('.*-merge')
  1900. self.waitUntilSettled()
  1901. # all jobs running
  1902. # Grab pointers to the jobs we want to release before
  1903. # releasing any, because list indexes may change as
  1904. # the jobs complete.
  1905. a, b, c = self.builds[:3]
  1906. a.release()
  1907. b.release()
  1908. c.release()
  1909. self.waitUntilSettled()
  1910. self.executor_server.hold_jobs_in_build = False
  1911. self.executor_server.release()
  1912. self.waitUntilSettled()
  1913. self.assertEqual(A.data['status'], 'NEW')
  1914. self.assertEqual(B.data['status'], 'MERGED')
  1915. self.assertEqual(C.data['status'], 'MERGED')
  1916. self.assertEqual(D.data['status'], 'MERGED')
  1917. self.assertEqual(E.data['status'], 'MERGED')
  1918. self.assertEqual(F.data['status'], 'MERGED')
  1919. self.assertEqual(A.reported, 2)
  1920. self.assertEqual(B.reported, 2)
  1921. self.assertEqual(C.reported, 2)
  1922. self.assertEqual(D.reported, 2)
  1923. self.assertEqual(E.reported, 2)
  1924. self.assertEqual(F.reported, 2)
  1925. self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 15)
  1926. self.assertEqual(len(self.history), 44)
  1927. def test_merger_repack(self):
  1928. "Test that the merger works after a repack"
  1929. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1930. A.addApproval('Code-Review', 2)
  1931. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  1932. self.waitUntilSettled()
  1933. self.assertEqual(self.getJobFromHistory('project-merge').result,
  1934. 'SUCCESS')
  1935. self.assertEqual(self.getJobFromHistory('project-test1').result,
  1936. 'SUCCESS')
  1937. self.assertEqual(self.getJobFromHistory('project-test2').result,
  1938. 'SUCCESS')
  1939. self.assertEqual(A.data['status'], 'MERGED')
  1940. self.assertEqual(A.reported, 2)
  1941. self.assertEmptyQueues()
  1942. self.build_history = []
  1943. path = os.path.join(self.merger_src_root, "review.example.com",
  1944. "org/project")
  1945. if os.path.exists(path):
  1946. repack_repo(path)
  1947. path = os.path.join(self.executor_src_root, "review.example.com",
  1948. "org/project")
  1949. if os.path.exists(path):
  1950. repack_repo(path)
  1951. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1952. A.addApproval('Code-Review', 2)
  1953. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  1954. self.waitUntilSettled()
  1955. self.assertEqual(self.getJobFromHistory('project-merge').result,
  1956. 'SUCCESS')
  1957. self.assertEqual(self.getJobFromHistory('project-test1').result,
  1958. 'SUCCESS')
  1959. self.assertEqual(self.getJobFromHistory('project-test2').result,
  1960. 'SUCCESS')
  1961. self.assertEqual(A.data['status'], 'MERGED')
  1962. self.assertEqual(A.reported, 2)
  1963. def test_merger_repack_large_change(self):
  1964. "Test that the merger works with large changes after a repack"
  1965. # https://bugs.executepad.net/zuul/+bug/1078946
  1966. # This test assumes the repo is already cloned; make sure it is
  1967. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  1968. trusted, project = tenant.getProject('org/project')
  1969. url = self.fake_gerrit.getGitUrl(project)
  1970. self.executor_server.merger._addProject('review.example.com',
  1971. 'org/project', url, None, None)
  1972. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  1973. A.addPatchset(large=True)
  1974. # TODOv3(jeblair): add hostname to upstream root
  1975. path = os.path.join(self.upstream_root, 'org/project')
  1976. repack_repo(path)
  1977. path = os.path.join(self.merger_src_root, 'review.example.com',
  1978. 'org/project')
  1979. if os.path.exists(path):
  1980. repack_repo(path)
  1981. path = os.path.join(self.executor_src_root, 'review.example.com',
  1982. 'org/project')
  1983. if os.path.exists(path):
  1984. repack_repo(path)
  1985. A.addApproval('Code-Review', 2)
  1986. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  1987. self.waitUntilSettled()
  1988. self.assertEqual(self.getJobFromHistory('project-merge').result,
  1989. 'SUCCESS')
  1990. self.assertEqual(self.getJobFromHistory('project-test1').result,
  1991. 'SUCCESS')
  1992. self.assertEqual(self.getJobFromHistory('project-test2').result,
  1993. 'SUCCESS')
  1994. self.assertEqual(A.data['status'], 'MERGED')
  1995. self.assertEqual(A.reported, 2)
  1996. def test_new_patchset_dequeues_old(self):
  1997. "Test that a new patchset causes the old to be dequeued"
  1998. # D -> C (depends on B) -> B (depends on A) -> A -> M
  1999. self.executor_server.hold_jobs_in_build = True
  2000. M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')
  2001. M.setMerged()
  2002. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2003. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  2004. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  2005. D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
  2006. A.addApproval('Code-Review', 2)
  2007. B.addApproval('Code-Review', 2)
  2008. C.addApproval('Code-Review', 2)
  2009. D.addApproval('Code-Review', 2)
  2010. C.setDependsOn(B, 1)
  2011. B.setDependsOn(A, 1)
  2012. A.setDependsOn(M, 1)
  2013. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2014. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  2015. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  2016. self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
  2017. self.waitUntilSettled()
  2018. B.addPatchset()
  2019. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
  2020. self.waitUntilSettled()
  2021. self.executor_server.hold_jobs_in_build = False
  2022. self.executor_server.release()
  2023. self.waitUntilSettled()
  2024. self.assertEqual(A.data['status'], 'MERGED')
  2025. self.assertEqual(A.reported, 2)
  2026. self.assertEqual(B.data['status'], 'NEW')
  2027. self.assertEqual(B.reported, 2)
  2028. self.assertEqual(C.data['status'], 'NEW')
  2029. self.assertEqual(C.reported, 2)
  2030. self.assertEqual(D.data['status'], 'MERGED')
  2031. self.assertEqual(D.reported, 2)
  2032. self.assertEqual(len(self.history), 9) # 3 each for A, B, D.
  2033. def test_new_patchset_check(self):
  2034. "Test a new patchset in check"
  2035. self.executor_server.hold_jobs_in_build = True
  2036. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2037. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  2038. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  2039. check_pipeline = tenant.layout.pipelines['check']
  2040. # Add two git-dependent changes
  2041. B.setDependsOn(A, 1)
  2042. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  2043. self.waitUntilSettled()
  2044. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  2045. self.waitUntilSettled()
  2046. # A live item, and a non-live/live pair
  2047. items = check_pipeline.getAllItems()
  2048. self.assertEqual(len(items), 3)
  2049. self.assertEqual(items[0].change.number, '1')
  2050. self.assertEqual(items[0].change.patchset, '1')
  2051. self.assertFalse(items[0].live)
  2052. self.assertEqual(items[1].change.number, '2')
  2053. self.assertEqual(items[1].change.patchset, '1')
  2054. self.assertTrue(items[1].live)
  2055. self.assertEqual(items[2].change.number, '1')
  2056. self.assertEqual(items[2].change.patchset, '1')
  2057. self.assertTrue(items[2].live)
  2058. # Add a new patchset to A
  2059. A.addPatchset()
  2060. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
  2061. self.waitUntilSettled()
  2062. # The live copy of A,1 should be gone, but the non-live and B
  2063. # should continue, and we should have a new A,2
  2064. items = check_pipeline.getAllItems()
  2065. self.assertEqual(len(items), 3)
  2066. self.assertEqual(items[0].change.number, '1')
  2067. self.assertEqual(items[0].change.patchset, '1')
  2068. self.assertFalse(items[0].live)
  2069. self.assertEqual(items[1].change.number, '2')
  2070. self.assertEqual(items[1].change.patchset, '1')
  2071. self.assertTrue(items[1].live)
  2072. self.assertEqual(items[2].change.number, '1')
  2073. self.assertEqual(items[2].change.patchset, '2')
  2074. self.assertTrue(items[2].live)
  2075. # Add a new patchset to B
  2076. B.addPatchset()
  2077. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
  2078. self.waitUntilSettled()
  2079. # The live copy of B,1 should be gone, and it's non-live copy of A,1
  2080. # but we should have a new B,2 (still based on A,1)
  2081. items = check_pipeline.getAllItems()
  2082. self.assertEqual(len(items), 3)
  2083. self.assertEqual(items[0].change.number, '1')
  2084. self.assertEqual(items[0].change.patchset, '2')
  2085. self.assertTrue(items[0].live)
  2086. self.assertEqual(items[1].change.number, '1')
  2087. self.assertEqual(items[1].change.patchset, '1')
  2088. self.assertFalse(items[1].live)
  2089. self.assertEqual(items[2].change.number, '2')
  2090. self.assertEqual(items[2].change.patchset, '2')
  2091. self.assertTrue(items[2].live)
  2092. self.builds[0].release()
  2093. self.waitUntilSettled()
  2094. self.builds[0].release()
  2095. self.waitUntilSettled()
  2096. self.executor_server.hold_jobs_in_build = False
  2097. self.executor_server.release()
  2098. self.waitUntilSettled()
  2099. self.assertEqual(A.reported, 1)
  2100. self.assertEqual(B.reported, 1)
  2101. self.assertEqual(self.history[0].result, 'ABORTED')
  2102. self.assertEqual(self.history[0].changes, '1,1')
  2103. self.assertEqual(self.history[1].result, 'ABORTED')
  2104. self.assertEqual(self.history[1].changes, '1,1 2,1')
  2105. self.assertEqual(self.history[2].result, 'SUCCESS')
  2106. self.assertEqual(self.history[2].changes, '1,2')
  2107. self.assertEqual(self.history[3].result, 'SUCCESS')
  2108. self.assertEqual(self.history[3].changes, '1,1 2,2')
  2109. def test_abandoned_gate(self):
  2110. "Test that an abandoned change is dequeued from gate"
  2111. self.executor_server.hold_jobs_in_build = True
  2112. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2113. A.addApproval('Code-Review', 2)
  2114. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2115. self.waitUntilSettled()
  2116. self.assertEqual(len(self.builds), 1, "One job being built (on hold)")
  2117. self.assertEqual(self.builds[0].name, 'project-merge')
  2118. self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
  2119. self.waitUntilSettled()
  2120. self.executor_server.release('.*-merge')
  2121. self.waitUntilSettled()
  2122. self.assertBuilds([])
  2123. self.assertHistory([
  2124. dict(name='project-merge', result='ABORTED', changes='1,1')],
  2125. ordered=False)
  2126. self.assertEqual(A.reported, 1,
  2127. "Abandoned gate change should report only start")
  2128. def test_abandoned_check(self):
  2129. "Test that an abandoned change is dequeued from check"
  2130. self.executor_server.hold_jobs_in_build = True
  2131. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2132. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  2133. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  2134. check_pipeline = tenant.layout.pipelines['check']
  2135. # Add two git-dependent changes
  2136. B.setDependsOn(A, 1)
  2137. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  2138. self.waitUntilSettled()
  2139. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  2140. self.waitUntilSettled()
  2141. # A live item, and a non-live/live pair
  2142. items = check_pipeline.getAllItems()
  2143. self.assertEqual(len(items), 3)
  2144. self.assertEqual(items[0].change.number, '1')
  2145. self.assertFalse(items[0].live)
  2146. self.assertEqual(items[1].change.number, '2')
  2147. self.assertTrue(items[1].live)
  2148. self.assertEqual(items[2].change.number, '1')
  2149. self.assertTrue(items[2].live)
  2150. # Abandon A
  2151. self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
  2152. self.waitUntilSettled()
  2153. # The live copy of A should be gone, but the non-live and B
  2154. # should continue
  2155. items = check_pipeline.getAllItems()
  2156. self.assertEqual(len(items), 2)
  2157. self.assertEqual(items[0].change.number, '1')
  2158. self.assertFalse(items[0].live)
  2159. self.assertEqual(items[1].change.number, '2')
  2160. self.assertTrue(items[1].live)
  2161. self.executor_server.hold_jobs_in_build = False
  2162. self.executor_server.release()
  2163. self.waitUntilSettled()
  2164. self.assertEqual(len(self.history), 4)
  2165. self.assertEqual(self.history[0].result, 'ABORTED',
  2166. 'Build should have been aborted')
  2167. self.assertEqual(A.reported, 0, "Abandoned change should not report")
  2168. self.assertEqual(B.reported, 1, "Change should report")
  2169. def test_cancel_starting_build(self):
  2170. "Test that a canceled build that is not processed yet is removed"
  2171. self.executor_server.hold_jobs_in_start = True
  2172. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2173. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  2174. for _ in iterate_timeout(30, 'Wait for build to be in starting phase'):
  2175. if self.executor_server.job_workers:
  2176. break
  2177. # Abandon change to cancel build
  2178. self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
  2179. for _ in iterate_timeout(30, 'Wait for executor:stop request'):
  2180. stop_jobs = [x for x in self.gearman_server.jobs_history
  2181. if b'executor:stop' in x.name]
  2182. if stop_jobs:
  2183. break
  2184. self.executor_server.hold_jobs_in_start = False
  2185. self.waitUntilSettled()
  2186. self.assertHistory([
  2187. dict(name='project-merge', result='ABORTED')
  2188. ])
  2189. def test_abandoned_not_timer(self):
  2190. "Test that an abandoned change does not cancel timer jobs"
  2191. # This test can not use simple_layout because it must start
  2192. # with a configuration which does not include a
  2193. # timer-triggered job so that we have an opportunity to set
  2194. # the hold flag before the first job.
  2195. self.executor_server.hold_jobs_in_build = True
  2196. # Start timer trigger - also org/project
  2197. self.commitConfigUpdate('common-config', 'layouts/idle.yaml')
  2198. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  2199. # The pipeline triggers every second, so we should have seen
  2200. # several by now.
  2201. time.sleep(5)
  2202. self.waitUntilSettled()
  2203. # Stop queuing timer triggered jobs so that the assertions
  2204. # below don't race against more jobs being queued.
  2205. # Must be in same repo, so overwrite config with another one
  2206. self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
  2207. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  2208. self.waitUntilSettled()
  2209. # If APScheduler is in mid-event when we remove the job, we
  2210. # can end up with one more event firing, so give it an extra
  2211. # second to settle.
  2212. time.sleep(1)
  2213. self.waitUntilSettled()
  2214. self.assertEqual(len(self.builds), 1, "One timer job")
  2215. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2216. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  2217. self.waitUntilSettled()
  2218. self.assertEqual(len(self.builds), 2, "One change plus one timer job")
  2219. self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
  2220. self.waitUntilSettled()
  2221. self.assertEqual(len(self.builds), 1, "One timer job remains")
  2222. self.executor_server.release()
  2223. self.waitUntilSettled()
  2224. def test_new_patchset_dequeues_old_on_head(self):
  2225. "Test that a new patchset causes the old to be dequeued (at head)"
  2226. # D -> C (depends on B) -> B (depends on A) -> A -> M
  2227. self.executor_server.hold_jobs_in_build = True
  2228. M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')
  2229. M.setMerged()
  2230. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2231. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  2232. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  2233. D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
  2234. A.addApproval('Code-Review', 2)
  2235. B.addApproval('Code-Review', 2)
  2236. C.addApproval('Code-Review', 2)
  2237. D.addApproval('Code-Review', 2)
  2238. C.setDependsOn(B, 1)
  2239. B.setDependsOn(A, 1)
  2240. A.setDependsOn(M, 1)
  2241. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2242. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  2243. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  2244. self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
  2245. self.waitUntilSettled()
  2246. A.addPatchset()
  2247. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
  2248. self.waitUntilSettled()
  2249. self.executor_server.hold_jobs_in_build = False
  2250. self.executor_server.release()
  2251. self.waitUntilSettled()
  2252. self.assertEqual(A.data['status'], 'NEW')
  2253. self.assertEqual(A.reported, 2)
  2254. self.assertEqual(B.data['status'], 'NEW')
  2255. self.assertEqual(B.reported, 2)
  2256. self.assertEqual(C.data['status'], 'NEW')
  2257. self.assertEqual(C.reported, 2)
  2258. self.assertEqual(D.data['status'], 'MERGED')
  2259. self.assertEqual(D.reported, 2)
  2260. self.assertEqual(len(self.history), 7)
  2261. def test_new_patchset_dequeues_old_without_dependents(self):
  2262. "Test that a new patchset causes only the old to be dequeued"
  2263. self.executor_server.hold_jobs_in_build = True
  2264. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2265. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  2266. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  2267. A.addApproval('Code-Review', 2)
  2268. B.addApproval('Code-Review', 2)
  2269. C.addApproval('Code-Review', 2)
  2270. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  2271. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  2272. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2273. self.waitUntilSettled()
  2274. B.addPatchset()
  2275. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
  2276. self.waitUntilSettled()
  2277. self.executor_server.hold_jobs_in_build = False
  2278. self.executor_server.release()
  2279. self.waitUntilSettled()
  2280. self.assertEqual(A.data['status'], 'MERGED')
  2281. self.assertEqual(A.reported, 2)
  2282. self.assertEqual(B.data['status'], 'NEW')
  2283. self.assertEqual(B.reported, 2)
  2284. self.assertEqual(C.data['status'], 'MERGED')
  2285. self.assertEqual(C.reported, 2)
  2286. self.assertEqual(len(self.history), 9)
  2287. def test_new_patchset_dequeues_old_independent_queue(self):
  2288. "Test that a new patchset causes the old to be dequeued (independent)"
  2289. self.executor_server.hold_jobs_in_build = True
  2290. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2291. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  2292. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  2293. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  2294. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  2295. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  2296. self.waitUntilSettled()
  2297. B.addPatchset()
  2298. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
  2299. self.waitUntilSettled()
  2300. self.executor_server.hold_jobs_in_build = False
  2301. self.executor_server.release()
  2302. self.waitUntilSettled()
  2303. self.assertEqual(A.data['status'], 'NEW')
  2304. self.assertEqual(A.reported, 1)
  2305. self.assertEqual(B.data['status'], 'NEW')
  2306. self.assertEqual(B.reported, 1)
  2307. self.assertEqual(C.data['status'], 'NEW')
  2308. self.assertEqual(C.reported, 1)
  2309. self.assertEqual(len(self.history), 10)
  2310. self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1)
  2311. @simple_layout('layouts/noop-job.yaml')
  2312. def test_noop_job(self):
  2313. "Test that the internal noop job works"
  2314. A = self.fake_gerrit.addFakeChange('org/noop-project', 'master', 'A')
  2315. A.addApproval('Code-Review', 2)
  2316. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2317. self.waitUntilSettled()
  2318. self.assertEqual(len(self.gearman_server.getQueue()), 0)
  2319. self.assertTrue(self.scheds.first.sched._areAllBuildsComplete())
  2320. self.assertEqual(len(self.history), 0)
  2321. self.assertEqual(A.data['status'], 'MERGED')
  2322. self.assertEqual(A.reported, 2)
  2323. @simple_layout('layouts/no-jobs-project.yaml')
  2324. def test_no_job_project(self):
  2325. "Test that reports with no jobs don't get sent"
  2326. A = self.fake_gerrit.addFakeChange('org/no-jobs-project',
  2327. 'master', 'A')
  2328. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  2329. self.waitUntilSettled()
  2330. # Change wasn't reported to
  2331. self.assertEqual(A.reported, False)
  2332. # Check queue is empty afterwards
  2333. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  2334. check_pipeline = tenant.layout.pipelines['check']
  2335. items = check_pipeline.getAllItems()
  2336. self.assertEqual(len(items), 0)
  2337. self.assertEqual(len(self.history), 0)
  2338. def test_zuul_refs(self):
  2339. "Test that zuul refs exist and have the right changes"
  2340. self.executor_server.hold_jobs_in_build = True
  2341. M1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'M1')
  2342. M1.setMerged()
  2343. M2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'M2')
  2344. M2.setMerged()
  2345. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  2346. B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
  2347. C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
  2348. D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
  2349. A.addApproval('Code-Review', 2)
  2350. B.addApproval('Code-Review', 2)
  2351. C.addApproval('Code-Review', 2)
  2352. D.addApproval('Code-Review', 2)
  2353. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2354. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  2355. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  2356. self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
  2357. self.waitUntilSettled()
  2358. self.executor_server.release('.*-merge')
  2359. self.waitUntilSettled()
  2360. self.executor_server.release('.*-merge')
  2361. self.waitUntilSettled()
  2362. self.executor_server.release('.*-merge')
  2363. self.waitUntilSettled()
  2364. self.executor_server.release('.*-merge')
  2365. self.waitUntilSettled()
  2366. a_build = b_build = c_build = d_build = None
  2367. for x in self.builds:
  2368. if x.parameters['zuul']['change'] == '3':
  2369. a_build = x
  2370. elif x.parameters['zuul']['change'] == '4':
  2371. b_build = x
  2372. elif x.parameters['zuul']['change'] == '5':
  2373. c_build = x
  2374. elif x.parameters['zuul']['change'] == '6':
  2375. d_build = x
  2376. if a_build and b_build and c_build and d_build:
  2377. break
  2378. # should have a, not b, and should not be in project2
  2379. self.assertTrue(a_build.hasChanges(A))
  2380. self.assertFalse(a_build.hasChanges(B, M2))
  2381. # should have a and b, and should not be in project2
  2382. self.assertTrue(b_build.hasChanges(A, B))
  2383. self.assertFalse(b_build.hasChanges(M2))
  2384. # should have a and b in 1, c in 2
  2385. self.assertTrue(c_build.hasChanges(A, B, C))
  2386. self.assertFalse(c_build.hasChanges(D))
  2387. # should have a and b in 1, c and d in 2
  2388. self.assertTrue(d_build.hasChanges(A, B, C, D))
  2389. self.executor_server.hold_jobs_in_build = False
  2390. self.executor_server.release()
  2391. self.waitUntilSettled()
  2392. self.assertEqual(A.data['status'], 'MERGED')
  2393. self.assertEqual(A.reported, 2)
  2394. self.assertEqual(B.data['status'], 'MERGED')
  2395. self.assertEqual(B.reported, 2)
  2396. self.assertEqual(C.data['status'], 'MERGED')
  2397. self.assertEqual(C.reported, 2)
  2398. self.assertEqual(D.data['status'], 'MERGED')
  2399. self.assertEqual(D.reported, 2)
  2400. def test_rerun_on_error(self):
  2401. "Test that if a worker fails to run a job, it is run again"
  2402. self.executor_server.hold_jobs_in_build = True
  2403. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2404. A.addApproval('Code-Review', 2)
  2405. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2406. self.waitUntilSettled()
  2407. self.builds[0].requeue = True
  2408. self.executor_server.hold_jobs_in_build = False
  2409. self.executor_server.release()
  2410. self.waitUntilSettled()
  2411. self.assertEqual(self.countJobResults(self.history, None), 1)
  2412. self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)
  2413. def test_statsd(self):
  2414. "Test each of the statsd methods used in the scheduler"
  2415. statsd = self.scheds.first.sched.statsd
  2416. statsd.incr('test-incr')
  2417. statsd.timing('test-timing', 3)
  2418. statsd.gauge('test-gauge', 12)
  2419. self.assertReportedStat('test-incr', '1', 'c')
  2420. self.assertReportedStat('test-timing', '3', 'ms')
  2421. self.assertReportedStat('test-gauge', '12', 'g')
  2422. # test key normalization
  2423. statsd.extra_keys = {'hostname': '1_2_3_4'}
  2424. statsd.incr('hostname-incr.{hostname}.{fake}', fake='1:2')
  2425. statsd.timing('hostname-timing.{hostname}.{fake}', 3, fake='1:2')
  2426. statsd.gauge('hostname-gauge.{hostname}.{fake}', 12, fake='1:2')
  2427. self.assertReportedStat('hostname-incr.1_2_3_4.1_2', '1', 'c')
  2428. self.assertReportedStat('hostname-timing.1_2_3_4.1_2', '3', 'ms')
  2429. self.assertReportedStat('hostname-gauge.1_2_3_4.1_2', '12', 'g')
  2430. def test_statsd_conflict(self):
  2431. statsd = self.scheds.first.sched.statsd
  2432. statsd.gauge('test-gauge', 12)
  2433. # since test-gauge is already a value, we can't make
  2434. # subvalues. Test the assert works.
  2435. statsd.gauge('test-gauge.1_2_3_4', 12)
  2436. self.assertReportedStat('test-gauge', '12', 'g')
  2437. self.assertRaises(Exception, self.assertReportedStat,
  2438. 'test-gauge.1_2_3_4', '12', 'g')
  2439. def test_stuck_job_cleanup(self):
  2440. "Test that pending jobs are cleaned up if removed from layout"
  2441. # We want to hold the project-merge job that the fake change enqueues
  2442. self.gearman_server.hold_jobs_in_queue = True
  2443. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2444. A.addApproval('Code-Review', 2)
  2445. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2446. self.waitUntilSettled()
  2447. # The assertion is that we have one job in the queue, project-merge
  2448. self.assertEqual(len(self.gearman_server.getQueue()), 1)
  2449. self.commitConfigUpdate('common-config', 'layouts/no-jobs.yaml')
  2450. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  2451. self.waitUntilSettled()
  2452. self.gearman_server.release('gate-noop')
  2453. self.waitUntilSettled()
  2454. # asserting that project-merge is removed from queue
  2455. self.assertEqual(len(self.gearman_server.getQueue()), 0)
  2456. self.assertTrue(self.scheds.first.sched._areAllBuildsComplete())
  2457. self.assertEqual(len(self.history), 1)
  2458. self.assertEqual(self.history[0].name, 'gate-noop')
  2459. self.assertEqual(self.history[0].result, 'SUCCESS')
  2460. def test_file_head(self):
  2461. # This is a regression test for an observed bug. A change
  2462. # with a file named "HEAD" in the root directory of the repo
  2463. # was processed by a merger. It then was unable to reset the
  2464. # repo because of:
  2465. # GitCommandError: 'git reset --hard HEAD' returned
  2466. # with exit code 128
  2467. # stderr: 'fatal: ambiguous argument 'HEAD': both revision
  2468. # and filename
  2469. # Use '--' to separate filenames from revisions'
  2470. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2471. A.addPatchset({'HEAD': ''})
  2472. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  2473. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
  2474. self.waitUntilSettled()
  2475. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  2476. self.waitUntilSettled()
  2477. self.assertIn('Build succeeded', A.messages[0])
  2478. self.assertIn('Build succeeded', B.messages[0])
  2479. def test_file_jobs(self):
  2480. "Test that file jobs run only when appropriate"
  2481. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2482. A.addPatchset({'pip-requires': 'foo'})
  2483. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  2484. A.addApproval('Code-Review', 2)
  2485. B.addApproval('Code-Review', 2)
  2486. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2487. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  2488. self.waitUntilSettled()
  2489. testfile_jobs = [x for x in self.history
  2490. if x.name == 'project-testfile']
  2491. self.assertEqual(len(testfile_jobs), 1)
  2492. self.assertEqual(testfile_jobs[0].changes, '1,2')
  2493. self.assertEqual(A.data['status'], 'MERGED')
  2494. self.assertEqual(A.reported, 2)
  2495. self.assertEqual(B.data['status'], 'MERGED')
  2496. self.assertEqual(B.reported, 2)
  2497. def _test_irrelevant_files_jobs(self, should_skip):
  2498. "Test that jobs with irrelevant-files filter run only when appropriate"
  2499. if should_skip:
  2500. files = {'ignoreme': 'ignored\n'}
  2501. else:
  2502. files = {'respectme': 'please!\n'}
  2503. change = self.fake_gerrit.addFakeChange('org/project',
  2504. 'master',
  2505. 'test irrelevant-files',
  2506. files=files)
  2507. self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
  2508. self.waitUntilSettled()
  2509. tested_change_ids = [x.changes[0] for x in self.history
  2510. if x.name == 'project-test-irrelevant-files']
  2511. if should_skip:
  2512. self.assertEqual([], tested_change_ids)
  2513. else:
  2514. self.assertIn(change.data['number'], tested_change_ids)
  2515. @simple_layout('layouts/irrelevant-files.yaml')
  2516. def test_irrelevant_files_match_skips_job(self):
  2517. self._test_irrelevant_files_jobs(should_skip=True)
  2518. @simple_layout('layouts/irrelevant-files.yaml')
  2519. def test_irrelevant_files_no_match_runs_job(self):
  2520. self._test_irrelevant_files_jobs(should_skip=False)
  2521. @simple_layout('layouts/inheritance.yaml')
  2522. def test_inherited_jobs_keep_matchers(self):
  2523. files = {'ignoreme': 'ignored\n'}
  2524. change = self.fake_gerrit.addFakeChange('org/project',
  2525. 'master',
  2526. 'test irrelevant-files',
  2527. files=files)
  2528. self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
  2529. self.waitUntilSettled()
  2530. run_jobs = set([build.name for build in self.history])
  2531. self.assertEqual(set(['project-test-nomatch-starts-empty',
  2532. 'project-test-nomatch-starts-full']), run_jobs)
  2533. @simple_layout('layouts/job-vars.yaml')
  2534. def test_inherited_job_variables(self):
  2535. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2536. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  2537. self.waitUntilSettled()
  2538. self.assertHistory([
  2539. dict(name='parentjob', result='SUCCESS'),
  2540. dict(name='child1', result='SUCCESS'),
  2541. dict(name='child2', result='SUCCESS'),
  2542. dict(name='child3', result='SUCCESS'),
  2543. dict(name='override_project_var', result='SUCCESS'),
  2544. dict(name='job_from_template1', result='SUCCESS'),
  2545. dict(name='job_from_template2', result='SUCCESS'),
  2546. ], ordered=False)
  2547. j = self.getJobFromHistory('parentjob')
  2548. rp = set([p['name'] for p in j.parameters['projects']])
  2549. self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project')
  2550. self.assertEqual(j.parameters['vars']['template_var1'],
  2551. 'set_in_template1')
  2552. self.assertEqual(j.parameters['vars']['template_var2'],
  2553. 'set_in_template2')
  2554. self.assertEqual(j.parameters['vars']['override'], 0)
  2555. self.assertEqual(j.parameters['vars']['child1override'], 0)
  2556. self.assertEqual(j.parameters['vars']['parent'], 0)
  2557. self.assertEqual(j.parameters['vars']['deep']['override'], 0)
  2558. self.assertFalse('child1' in j.parameters['vars'])
  2559. self.assertFalse('child2' in j.parameters['vars'])
  2560. self.assertFalse('child3' in j.parameters['vars'])
  2561. self.assertEqual(rp, set(['org/project', 'org/project0',
  2562. 'org/project0']))
  2563. j = self.getJobFromHistory('child1')
  2564. rp = set([p['name'] for p in j.parameters['projects']])
  2565. self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project')
  2566. self.assertEqual(j.parameters['vars']['override'], 1)
  2567. self.assertEqual(j.parameters['vars']['child1override'], 1)
  2568. self.assertEqual(j.parameters['vars']['parent'], 0)
  2569. self.assertEqual(j.parameters['vars']['child1'], 1)
  2570. self.assertEqual(j.parameters['vars']['deep']['override'], 1)
  2571. self.assertFalse('child2' in j.parameters['vars'])
  2572. self.assertFalse('child3' in j.parameters['vars'])
  2573. self.assertEqual(rp, set(['org/project', 'org/project0',
  2574. 'org/project1']))
  2575. j = self.getJobFromHistory('child2')
  2576. self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project')
  2577. rp = set([p['name'] for p in j.parameters['projects']])
  2578. self.assertEqual(j.parameters['vars']['override'], 2)
  2579. self.assertEqual(j.parameters['vars']['child1override'], 0)
  2580. self.assertEqual(j.parameters['vars']['parent'], 0)
  2581. self.assertEqual(j.parameters['vars']['deep']['override'], 2)
  2582. self.assertFalse('child1' in j.parameters['vars'])
  2583. self.assertEqual(j.parameters['vars']['child2'], 2)
  2584. self.assertFalse('child3' in j.parameters['vars'])
  2585. self.assertEqual(rp, set(['org/project', 'org/project0',
  2586. 'org/project2']))
  2587. j = self.getJobFromHistory('child3')
  2588. self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project')
  2589. rp = set([p['name'] for p in j.parameters['projects']])
  2590. self.assertEqual(j.parameters['vars']['override'], 3)
  2591. self.assertEqual(j.parameters['vars']['child1override'], 0)
  2592. self.assertEqual(j.parameters['vars']['parent'], 0)
  2593. self.assertEqual(j.parameters['vars']['deep']['override'], 3)
  2594. self.assertFalse('child1' in j.parameters['vars'])
  2595. self.assertFalse('child2' in j.parameters['vars'])
  2596. self.assertEqual(j.parameters['vars']['child3'], 3)
  2597. self.assertEqual(rp, set(['org/project', 'org/project0',
  2598. 'org/project3']))
  2599. j = self.getJobFromHistory('override_project_var')
  2600. self.assertEqual(j.parameters['vars']['project_var'],
  2601. 'override_in_job')
  2602. @simple_layout('layouts/job-variants.yaml')
  2603. def test_job_branch_variants(self):
  2604. self.create_branch('org/project', 'stable/diablo')
  2605. self.fake_gerrit.addEvent(
  2606. self.fake_gerrit.getFakeBranchCreatedEvent(
  2607. 'org/project', 'stable/diablo'))
  2608. self.create_branch('org/project', 'stable/essex')
  2609. self.fake_gerrit.addEvent(
  2610. self.fake_gerrit.getFakeBranchCreatedEvent(
  2611. 'org/project', 'stable/essex'))
  2612. self.waitUntilSettled()
  2613. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2614. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  2615. self.waitUntilSettled()
  2616. B = self.fake_gerrit.addFakeChange('org/project', 'stable/diablo', 'B')
  2617. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  2618. self.waitUntilSettled()
  2619. C = self.fake_gerrit.addFakeChange('org/project', 'stable/essex', 'C')
  2620. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  2621. self.waitUntilSettled()
  2622. self.assertHistory([
  2623. dict(name='python27', result='SUCCESS'),
  2624. dict(name='python27', result='SUCCESS'),
  2625. dict(name='python27', result='SUCCESS'),
  2626. ])
  2627. p = self.history[0].parameters
  2628. self.assertEqual(p['timeout'], 40)
  2629. self.assertEqual(len(p['nodes']), 1)
  2630. self.assertEqual(p['nodes'][0]['label'], 'new')
  2631. self.assertEqual([x['path'] for x in p['pre_playbooks']],
  2632. ['base-pre', 'py27-pre'])
  2633. self.assertEqual([x['path'] for x in p['post_playbooks']],
  2634. ['py27-post-a', 'py27-post-b', 'base-post'])
  2635. self.assertEqual([x['path'] for x in p['playbooks']],
  2636. ['playbooks/python27.yaml'])
  2637. p = self.history[1].parameters
  2638. self.assertEqual(p['timeout'], 50)
  2639. self.assertEqual(len(p['nodes']), 1)
  2640. self.assertEqual(p['nodes'][0]['label'], 'old')
  2641. self.assertEqual([x['path'] for x in p['pre_playbooks']],
  2642. ['base-pre', 'py27-pre', 'py27-diablo-pre'])
  2643. self.assertEqual([x['path'] for x in p['post_playbooks']],
  2644. ['py27-diablo-post', 'py27-post-a', 'py27-post-b',
  2645. 'base-post'])
  2646. self.assertEqual([x['path'] for x in p['playbooks']],
  2647. ['py27-diablo'])
  2648. p = self.history[2].parameters
  2649. self.assertEqual(p['timeout'], 40)
  2650. self.assertEqual(len(p['nodes']), 1)
  2651. self.assertEqual(p['nodes'][0]['label'], 'new')
  2652. self.assertEqual([x['path'] for x in p['pre_playbooks']],
  2653. ['base-pre', 'py27-pre', 'py27-essex-pre'])
  2654. self.assertEqual([x['path'] for x in p['post_playbooks']],
  2655. ['py27-essex-post', 'py27-post-a', 'py27-post-b',
  2656. 'base-post'])
  2657. self.assertEqual([x['path'] for x in p['playbooks']],
  2658. ['playbooks/python27.yaml'])
  2659. @simple_layout("layouts/no-run.yaml")
  2660. def test_job_without_run(self):
  2661. "Test that a job without a run playbook errors"
  2662. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2663. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  2664. self.waitUntilSettled()
  2665. self.assertIn('Job base does not specify a run playbook',
  2666. A.messages[-1])
  2667. def test_queue_names(self):
  2668. "Test shared change queue names"
  2669. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  2670. (trusted, project1) = tenant.getProject('org/project1')
  2671. (trusted, project2) = tenant.getProject('org/project2')
  2672. q1 = tenant.layout.pipelines['gate'].getQueue(project1)
  2673. q2 = tenant.layout.pipelines['gate'].getQueue(project2)
  2674. self.assertEqual(q1.name, 'integrated')
  2675. self.assertEqual(q2.name, 'integrated')
  2676. @simple_layout("layouts/template-queue.yaml")
  2677. def test_template_queue(self):
  2678. "Test a shared queue can be constructed from a project-template"
  2679. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  2680. (trusted, project1) = tenant.getProject('org/project1')
  2681. (trusted, project2) = tenant.getProject('org/project2')
  2682. q1 = tenant.layout.pipelines['gate'].getQueue(project1)
  2683. q2 = tenant.layout.pipelines['gate'].getQueue(project2)
  2684. self.assertEqual(q1.name, 'integrated')
  2685. self.assertEqual(q2.name, 'integrated')
  2686. @simple_layout("layouts/regex-template-queue.yaml")
  2687. def test_regex_template_queue(self):
  2688. "Test a shared queue can be constructed from a regex project-template"
  2689. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  2690. (trusted, project1) = tenant.getProject('org/project1')
  2691. (trusted, project2) = tenant.getProject('org/project2')
  2692. q1 = tenant.layout.pipelines['gate'].getQueue(project1)
  2693. q2 = tenant.layout.pipelines['gate'].getQueue(project2)
  2694. self.assertEqual(q1.name, 'integrated')
  2695. self.assertEqual(q2.name, 'integrated')
  2696. @simple_layout("layouts/regex-queue.yaml")
  2697. def test_regex_queue(self):
  2698. "Test a shared queue can be constructed from a regex project"
  2699. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  2700. (trusted, project1) = tenant.getProject('org/project1')
  2701. (trusted, project2) = tenant.getProject('org/project2')
  2702. q1 = tenant.layout.pipelines['gate'].getQueue(project1)
  2703. q2 = tenant.layout.pipelines['gate'].getQueue(project2)
  2704. self.assertEqual(q1.name, 'integrated')
  2705. self.assertEqual(q2.name, 'integrated')
  2706. def test_queue_precedence(self):
  2707. "Test that queue precedence works"
  2708. self.gearman_server.hold_jobs_in_queue = True
  2709. self.executor_server.hold_jobs_in_build = True
  2710. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2711. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  2712. A.addApproval('Code-Review', 2)
  2713. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2714. self.waitUntilSettled()
  2715. self.gearman_server.hold_jobs_in_queue = False
  2716. self.gearman_server.release()
  2717. self.waitUntilSettled()
  2718. # Run one build at a time to ensure non-race order:
  2719. self.orderedRelease()
  2720. self.executor_server.hold_jobs_in_build = False
  2721. self.waitUntilSettled()
  2722. self.log.debug(self.history)
  2723. self.assertEqual(self.history[0].pipeline, 'gate')
  2724. self.assertEqual(self.history[1].pipeline, 'check')
  2725. self.assertEqual(self.history[2].pipeline, 'gate')
  2726. self.assertEqual(self.history[3].pipeline, 'gate')
  2727. self.assertEqual(self.history[4].pipeline, 'check')
  2728. self.assertEqual(self.history[5].pipeline, 'check')
  2729. def test_reconfigure_merge(self):
  2730. """Test that two reconfigure events are merged"""
  2731. tenant = self.scheds.first.sched.abide.tenants['tenant-one']
  2732. (trusted, project) = tenant.getProject('org/project')
  2733. self.scheds.first.sched.run_handler_lock.acquire()
  2734. self.assertEqual(
  2735. self.scheds.first.sched.management_event_queue.qsize(), 0)
  2736. self.scheds.first.sched.reconfigureTenant(tenant, project, None)
  2737. self.assertEqual(
  2738. self.scheds.first.sched.management_event_queue.qsize(), 1)
  2739. self.scheds.first.sched.reconfigureTenant(tenant, project, None)
  2740. # The second event should have been combined with the first
  2741. # so we should still only have one entry.
  2742. self.assertEqual(
  2743. self.scheds.first.sched.management_event_queue.qsize(), 1)
  2744. self.scheds.first.sched.run_handler_lock.release()
  2745. self.waitUntilSettled()
  2746. self.assertEqual(
  2747. self.scheds.first.sched.management_event_queue.qsize(), 0)
  2748. def test_live_reconfiguration(self):
  2749. "Test that live reconfiguration works"
  2750. self.executor_server.hold_jobs_in_build = True
  2751. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2752. A.addApproval('Code-Review', 2)
  2753. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2754. self.waitUntilSettled()
  2755. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  2756. self.waitUntilSettled()
  2757. self.executor_server.hold_jobs_in_build = False
  2758. self.executor_server.release()
  2759. self.waitUntilSettled()
  2760. self.assertEqual(self.getJobFromHistory('project-merge').result,
  2761. 'SUCCESS')
  2762. self.assertEqual(self.getJobFromHistory('project-test1').result,
  2763. 'SUCCESS')
  2764. self.assertEqual(self.getJobFromHistory('project-test2').result,
  2765. 'SUCCESS')
  2766. self.assertEqual(A.data['status'], 'MERGED')
  2767. self.assertEqual(A.reported, 2)
  2768. def test_live_reconfiguration_command_socket(self):
  2769. "Test that live reconfiguration via command socket works"
  2770. # record previous tenant reconfiguration time, which may not be set
  2771. old = self.scheds.first.sched.tenant_last_reconfigured\
  2772. .get('tenant-one', 0)
  2773. self.waitUntilSettled()
  2774. command_socket = self.config.get('scheduler', 'command_socket')
  2775. with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
  2776. s.connect(command_socket)
  2777. s.sendall('full-reconfigure\n'.encode('utf8'))
  2778. # Wait for full reconfiguration. Note that waitUntilSettled is not
  2779. # reliable here because the reconfigure event may arrive in the
  2780. # event queue after waitUntilSettled.
  2781. start = time.time()
  2782. while True:
  2783. if time.time() - start > 15:
  2784. raise Exception("Timeout waiting for full reconfiguration")
  2785. new = self.scheds.first.sched.tenant_last_reconfigured\
  2786. .get('tenant-one', 0)
  2787. if old < new:
  2788. break
  2789. else:
  2790. time.sleep(0)
  2791. def test_live_reconfiguration_abort(self):
  2792. # Raise an exception during reconfiguration and verify we
  2793. # still function.
  2794. self.executor_server.hold_jobs_in_build = True
  2795. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2796. A.addApproval('Code-Review', 2)
  2797. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2798. self.waitUntilSettled()
  2799. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  2800. pipeline = tenant.layout.pipelines['gate']
  2801. change = pipeline.getAllItems()[0].change
  2802. # Set this to an invalid value to cause an exception during
  2803. # reconfiguration.
  2804. change.branch = None
  2805. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  2806. self.waitUntilSettled()
  2807. self.executor_server.hold_jobs_in_build = False
  2808. self.executor_server.release()
  2809. self.waitUntilSettled()
  2810. self.assertEqual(self.getJobFromHistory('project-merge').result,
  2811. 'ABORTED')
  2812. self.assertEqual(A.data['status'], 'NEW')
  2813. # The final report fails because of the invalid value set above.
  2814. self.assertEqual(A.reported, 1)
  2815. def test_live_reconfiguration_merge_conflict(self):
  2816. # A real-world bug: a change in a gate queue has a merge
  2817. # conflict and a job is added to its project while it's
  2818. # sitting in the queue. The job gets added to the change and
  2819. # enqueued and the change gets stuck.
  2820. self.executor_server.hold_jobs_in_build = True
  2821. # This change is fine. It's here to stop the queue long
  2822. # enough for the next change to be subject to the
  2823. # reconfiguration, as well as to provide a conflict for the
  2824. # next change. This change will succeed and merge.
  2825. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2826. A.addPatchset({'conflict': 'A'})
  2827. A.addApproval('Code-Review', 2)
  2828. # This change will be in merge conflict. During the
  2829. # reconfiguration, we will add a job. We want to make sure
  2830. # that doesn't cause it to get stuck.
  2831. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  2832. B.addPatchset({'conflict': 'B'})
  2833. B.addApproval('Code-Review', 2)
  2834. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2835. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  2836. self.waitUntilSettled()
  2837. # No jobs have run yet
  2838. self.assertEqual(A.data['status'], 'NEW')
  2839. self.assertEqual(A.reported, 1)
  2840. self.assertEqual(B.data['status'], 'NEW')
  2841. self.assertEqual(len(self.history), 0)
  2842. # Add the "project-test3" job.
  2843. self.commitConfigUpdate('common-config',
  2844. 'layouts/live-reconfiguration-add-job.yaml')
  2845. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  2846. self.waitUntilSettled()
  2847. self.executor_server.hold_jobs_in_build = False
  2848. self.executor_server.release()
  2849. self.waitUntilSettled()
  2850. self.assertEqual(A.data['status'], 'MERGED')
  2851. self.assertEqual(A.reported, 2)
  2852. self.assertEqual(B.data['status'], 'NEW')
  2853. self.assertIn('Merge Failed', B.messages[-1])
  2854. self.assertEqual(self.getJobFromHistory('project-merge').result,
  2855. 'SUCCESS')
  2856. self.assertEqual(self.getJobFromHistory('project-test1').result,
  2857. 'SUCCESS')
  2858. self.assertEqual(self.getJobFromHistory('project-test2').result,
  2859. 'SUCCESS')
  2860. self.assertEqual(self.getJobFromHistory('project-test3').result,
  2861. 'SUCCESS')
  2862. self.assertEqual(len(self.history), 4)
  2863. def test_live_reconfiguration_failed_root(self):
  2864. # An extrapolation of test_live_reconfiguration_merge_conflict
  2865. # that tests a job added to a job tree with a failed root does
  2866. # not run.
  2867. self.executor_server.hold_jobs_in_build = True
  2868. # This change is fine. It's here to stop the queue long
  2869. # enough for the next change to be subject to the
  2870. # reconfiguration. This change will succeed and merge.
  2871. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2872. A.addPatchset({'conflict': 'A'})
  2873. A.addApproval('Code-Review', 2)
  2874. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  2875. self.waitUntilSettled()
  2876. self.executor_server.release('.*-merge')
  2877. self.waitUntilSettled()
  2878. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  2879. self.executor_server.failJob('project-merge', B)
  2880. B.addApproval('Code-Review', 2)
  2881. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  2882. self.waitUntilSettled()
  2883. self.executor_server.release('.*-merge')
  2884. self.waitUntilSettled()
  2885. # Both -merge jobs have run, but no others.
  2886. self.assertEqual(A.data['status'], 'NEW')
  2887. self.assertEqual(A.reported, 1)
  2888. self.assertEqual(B.data['status'], 'NEW')
  2889. self.assertEqual(B.reported, 1)
  2890. self.assertEqual(self.history[0].result, 'SUCCESS')
  2891. self.assertEqual(self.history[0].name, 'project-merge')
  2892. self.assertEqual(self.history[1].result, 'FAILURE')
  2893. self.assertEqual(self.history[1].name, 'project-merge')
  2894. self.assertEqual(len(self.history), 2)
  2895. # Add the "project-test3" job.
  2896. self.commitConfigUpdate('common-config',
  2897. 'layouts/live-reconfiguration-add-job.yaml')
  2898. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  2899. self.waitUntilSettled()
  2900. self.executor_server.hold_jobs_in_build = False
  2901. self.executor_server.release()
  2902. self.waitUntilSettled()
  2903. self.assertEqual(A.data['status'], 'MERGED')
  2904. self.assertEqual(A.reported, 2)
  2905. self.assertEqual(B.data['status'], 'NEW')
  2906. self.assertEqual(B.reported, 2)
  2907. self.assertEqual(self.history[0].result, 'SUCCESS')
  2908. self.assertEqual(self.history[0].name, 'project-merge')
  2909. self.assertEqual(self.history[1].result, 'FAILURE')
  2910. self.assertEqual(self.history[1].name, 'project-merge')
  2911. self.assertEqual(self.history[2].result, 'SUCCESS')
  2912. self.assertEqual(self.history[3].result, 'SUCCESS')
  2913. self.assertEqual(self.history[4].result, 'SUCCESS')
  2914. self.assertEqual(len(self.history), 5)
  2915. def test_live_reconfiguration_failed_job(self):
  2916. # Test that a change with a removed failing job does not
  2917. # disrupt reconfiguration. If a change has a failed job and
  2918. # that job is removed during a reconfiguration, we observed a
  2919. # bug where the code to re-set build statuses would run on
  2920. # that build and raise an exception because the job no longer
  2921. # existed.
  2922. self.executor_server.hold_jobs_in_build = True
  2923. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  2924. # This change will fail and later be removed by the reconfiguration.
  2925. self.executor_server.failJob('project-test1', A)
  2926. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  2927. self.waitUntilSettled()
  2928. self.executor_server.release('.*-merge')
  2929. self.waitUntilSettled()
  2930. self.executor_server.release('project-test1')
  2931. self.waitUntilSettled()
  2932. self.assertEqual(A.data['status'], 'NEW')
  2933. self.assertEqual(A.reported, 0)
  2934. self.assertEqual(self.getJobFromHistory('project-merge').result,
  2935. 'SUCCESS')
  2936. self.assertEqual(self.getJobFromHistory('project-test1').result,
  2937. 'FAILURE')
  2938. self.assertEqual(len(self.history), 2)
  2939. # Remove the test1 job.
  2940. self.commitConfigUpdate('common-config',
  2941. 'layouts/live-reconfiguration-failed-job.yaml')
  2942. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  2943. self.waitUntilSettled()
  2944. self.executor_server.hold_jobs_in_build = False
  2945. self.executor_server.release()
  2946. self.waitUntilSettled()
  2947. self.assertEqual(self.getJobFromHistory('project-test2').result,
  2948. 'SUCCESS')
  2949. self.assertEqual(self.getJobFromHistory('project-testfile').result,
  2950. 'SUCCESS')
  2951. self.assertEqual(len(self.history), 4)
  2952. self.assertEqual(A.data['status'], 'NEW')
  2953. self.assertEqual(A.reported, 1)
  2954. self.assertIn('Build succeeded', A.messages[0])
  2955. # Ensure the removed job was not included in the report.
  2956. self.assertNotIn('project-test1', A.messages[0])
  2957. def test_live_reconfiguration_shared_queue(self):
  2958. # Test that a change with a failing job which was removed from
  2959. # this project but otherwise still exists in the system does
  2960. # not disrupt reconfiguration.
  2961. self.executor_server.hold_jobs_in_build = True
  2962. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  2963. self.executor_server.failJob('project1-project2-integration', A)
  2964. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  2965. self.waitUntilSettled()
  2966. self.executor_server.release('.*-merge')
  2967. self.waitUntilSettled()
  2968. self.executor_server.release('project1-project2-integration')
  2969. self.waitUntilSettled()
  2970. self.assertEqual(A.data['status'], 'NEW')
  2971. self.assertEqual(A.reported, 0)
  2972. self.assertEqual(self.getJobFromHistory('project-merge').result,
  2973. 'SUCCESS')
  2974. self.assertEqual(self.getJobFromHistory(
  2975. 'project1-project2-integration').result, 'FAILURE')
  2976. self.assertEqual(len(self.history), 2)
  2977. # Remove the integration job.
  2978. self.commitConfigUpdate(
  2979. 'common-config',
  2980. 'layouts/live-reconfiguration-shared-queue.yaml')
  2981. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  2982. self.waitUntilSettled()
  2983. self.executor_server.hold_jobs_in_build = False
  2984. self.executor_server.release()
  2985. self.waitUntilSettled()
  2986. self.assertEqual(self.getJobFromHistory('project-merge').result,
  2987. 'SUCCESS')
  2988. self.assertEqual(self.getJobFromHistory('project-test1').result,
  2989. 'SUCCESS')
  2990. self.assertEqual(self.getJobFromHistory('project-test2').result,
  2991. 'SUCCESS')
  2992. self.assertEqual(self.getJobFromHistory(
  2993. 'project1-project2-integration').result, 'FAILURE')
  2994. self.assertEqual(len(self.history), 4)
  2995. self.assertEqual(A.data['status'], 'NEW')
  2996. self.assertEqual(A.reported, 1)
  2997. self.assertIn('Build succeeded', A.messages[0])
  2998. # Ensure the removed job was not included in the report.
  2999. self.assertNotIn('project1-project2-integration', A.messages[0])
  3000. def test_live_reconfiguration_shared_queue_removed(self):
  3001. # Test that changes in a shared queue survive a change of the
  3002. # queue during reconfiguration. This is a regression test
  3003. # for the dependent pipeline manager.
  3004. self.executor_server.hold_jobs_in_build = True
  3005. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  3006. B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
  3007. A.addApproval('Code-Review', 2)
  3008. B.addApproval('Code-Review', 2)
  3009. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  3010. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  3011. self.waitUntilSettled()
  3012. # Remove the integration job.
  3013. self.commitConfigUpdate(
  3014. 'common-config',
  3015. 'layouts/live-reconfiguration-shared-queue-removed.yaml')
  3016. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3017. self.waitUntilSettled()
  3018. self.executor_server.hold_jobs_in_build = False
  3019. self.executor_server.release()
  3020. self.waitUntilSettled()
  3021. self.assertEqual(A.data['status'], 'MERGED')
  3022. self.assertEqual(B.data['status'], 'MERGED')
  3023. def test_double_live_reconfiguration_shared_queue(self):
  3024. # This was a real-world regression. A change is added to
  3025. # gate; a reconfigure happens, a second change which depends
  3026. # on the first is added, and a second reconfiguration happens.
  3027. # Ensure that both changes merge.
  3028. # A failure may indicate incorrect caching or cleaning up of
  3029. # references during a reconfiguration.
  3030. self.executor_server.hold_jobs_in_build = True
  3031. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  3032. B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
  3033. B.setDependsOn(A, 1)
  3034. A.addApproval('Code-Review', 2)
  3035. B.addApproval('Code-Review', 2)
  3036. # Add the parent change.
  3037. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  3038. self.waitUntilSettled()
  3039. self.executor_server.release('.*-merge')
  3040. self.waitUntilSettled()
  3041. # Reconfigure (with only one change in the pipeline).
  3042. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3043. self.waitUntilSettled()
  3044. # Add the child change.
  3045. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  3046. self.waitUntilSettled()
  3047. self.executor_server.release('.*-merge')
  3048. self.waitUntilSettled()
  3049. # Reconfigure (with both in the pipeline).
  3050. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3051. self.waitUntilSettled()
  3052. self.executor_server.hold_jobs_in_build = False
  3053. self.executor_server.release()
  3054. self.waitUntilSettled()
  3055. self.assertEqual(len(self.history), 8)
  3056. self.assertEqual(A.data['status'], 'MERGED')
  3057. self.assertEqual(A.reported, 2)
  3058. self.assertEqual(B.data['status'], 'MERGED')
  3059. self.assertEqual(B.reported, 2)
  3060. def test_live_reconfiguration_del_project(self):
  3061. # Test project deletion from layout
  3062. # while changes are enqueued
  3063. self.executor_server.hold_jobs_in_build = True
  3064. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3065. B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
  3066. C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')
  3067. # A Depends-On: B
  3068. A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
  3069. A.subject, B.data['id'])
  3070. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  3071. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  3072. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  3073. self.waitUntilSettled()
  3074. self.executor_server.release('.*-merge')
  3075. self.waitUntilSettled()
  3076. self.assertEqual(len(self.builds), 5)
  3077. # This layout defines only org/project, not org/project1
  3078. self.commitConfigUpdate(
  3079. 'common-config',
  3080. 'layouts/live-reconfiguration-del-project.yaml')
  3081. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3082. self.waitUntilSettled()
  3083. # Builds for C aborted, builds for A succeed,
  3084. # and have change B applied ahead
  3085. job_c = self.getJobFromHistory('project-test1')
  3086. self.assertEqual(job_c.changes, '3,1')
  3087. self.assertEqual(job_c.result, 'ABORTED')
  3088. self.executor_server.hold_jobs_in_build = False
  3089. self.executor_server.release()
  3090. self.waitUntilSettled()
  3091. self.assertEqual(
  3092. self.getJobFromHistory('project-test1', 'org/project').changes,
  3093. '2,1 1,1')
  3094. self.assertEqual(A.data['status'], 'NEW')
  3095. self.assertEqual(B.data['status'], 'NEW')
  3096. self.assertEqual(C.data['status'], 'NEW')
  3097. self.assertEqual(A.reported, 1)
  3098. self.assertEqual(B.reported, 0)
  3099. self.assertEqual(C.reported, 0)
  3100. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  3101. self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
  3102. self.assertIn('Build succeeded', A.messages[0])
  3103. @simple_layout("layouts/reconfigure-failed-head.yaml")
  3104. def test_live_reconfiguration_failed_change_at_head(self):
  3105. # Test that if we reconfigure with a failed change at head,
  3106. # that the change behind it isn't reparented onto it.
  3107. self.executor_server.hold_jobs_in_build = True
  3108. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3109. A.addApproval('Code-Review', 2)
  3110. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  3111. B.addApproval('Code-Review', 2)
  3112. self.executor_server.failJob('job1', A)
  3113. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  3114. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  3115. self.waitUntilSettled()
  3116. self.assertBuilds([
  3117. dict(name='job1', changes='1,1'),
  3118. dict(name='job2', changes='1,1'),
  3119. dict(name='job1', changes='1,1 2,1'),
  3120. dict(name='job2', changes='1,1 2,1'),
  3121. ])
  3122. self.release(self.builds[0])
  3123. self.waitUntilSettled()
  3124. self.assertBuilds([
  3125. dict(name='job2', changes='1,1'),
  3126. dict(name='job1', changes='2,1'),
  3127. dict(name='job2', changes='2,1'),
  3128. ])
  3129. # Unordered history comparison because the aborts can finish
  3130. # in any order.
  3131. self.assertHistory([
  3132. dict(name='job1', result='FAILURE', changes='1,1'),
  3133. dict(name='job1', result='ABORTED', changes='1,1 2,1'),
  3134. dict(name='job2', result='ABORTED', changes='1,1 2,1'),
  3135. ], ordered=False)
  3136. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3137. self.waitUntilSettled()
  3138. self.executor_server.hold_jobs_in_build = False
  3139. self.executor_server.release()
  3140. self.waitUntilSettled()
  3141. self.assertBuilds([])
  3142. self.assertHistory([
  3143. dict(name='job1', result='FAILURE', changes='1,1'),
  3144. dict(name='job1', result='ABORTED', changes='1,1 2,1'),
  3145. dict(name='job2', result='ABORTED', changes='1,1 2,1'),
  3146. dict(name='job2', result='SUCCESS', changes='1,1'),
  3147. dict(name='job1', result='SUCCESS', changes='2,1'),
  3148. dict(name='job2', result='SUCCESS', changes='2,1'),
  3149. ], ordered=False)
  3150. self.assertEqual(A.data['status'], 'NEW')
  3151. self.assertEqual(B.data['status'], 'MERGED')
  3152. self.assertEqual(A.reported, 2)
  3153. self.assertEqual(B.reported, 2)
  3154. def test_delayed_repo_init(self):
  3155. self.init_repo("org/new-project")
  3156. files = {'README': ''}
  3157. self.addCommitToRepo("org/new-project", 'Initial commit',
  3158. files=files, tag='init')
  3159. self.newTenantConfig('tenants/delayed-repo-init.yaml')
  3160. self.commitConfigUpdate(
  3161. 'common-config',
  3162. 'layouts/delayed-repo-init.yaml')
  3163. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3164. self.waitUntilSettled()
  3165. A = self.fake_gerrit.addFakeChange('org/new-project', 'master', 'A')
  3166. A.addApproval('Code-Review', 2)
  3167. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  3168. self.waitUntilSettled()
  3169. self.assertEqual(self.getJobFromHistory('project-merge').result,
  3170. 'SUCCESS')
  3171. self.assertEqual(self.getJobFromHistory('project-test1').result,
  3172. 'SUCCESS')
  3173. self.assertEqual(self.getJobFromHistory('project-test2').result,
  3174. 'SUCCESS')
  3175. self.assertEqual(A.data['status'], 'MERGED')
  3176. self.assertEqual(A.reported, 2)
  3177. @simple_layout('layouts/single-job-with-nodeset.yaml')
  3178. def test_live_reconfiguration_queued_node_requests(self):
  3179. # Test that a job with a queued node request still has the
  3180. # correct state after reconfiguration.
  3181. self.fake_nodepool.pause()
  3182. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3183. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  3184. self.waitUntilSettled()
  3185. def get_job():
  3186. data = json.loads(self.scheds.first.sched
  3187. .formatStatusJSON('tenant-one'))
  3188. for pipeline in data['pipelines']:
  3189. for queue in pipeline['change_queues']:
  3190. for head in queue['heads']:
  3191. for item in head:
  3192. for job in item['jobs']:
  3193. if job['name'] == 'check-job':
  3194. return job
  3195. job = get_job()
  3196. self.assertTrue(job['queued'])
  3197. self.scheds.execute(lambda app: app.sched.reconfigure(self.config))
  3198. self.waitUntilSettled()
  3199. job = get_job()
  3200. self.assertTrue(job['queued'])
  3201. self.fake_nodepool.unpause()
  3202. self.waitUntilSettled()
  3203. self.assertHistory([
  3204. dict(name='check-job', result='SUCCESS', changes='1,1'),
  3205. ])
  3206. @simple_layout('layouts/repo-deleted.yaml')
  3207. def test_repo_deleted(self):
  3208. self.init_repo("org/delete-project")
  3209. A = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'A')
  3210. A.addApproval('Code-Review', 2)
  3211. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  3212. self.waitUntilSettled()
  3213. self.assertEqual(self.getJobFromHistory('project-merge').result,
  3214. 'SUCCESS')
  3215. self.assertEqual(self.getJobFromHistory('project-test1').result,
  3216. 'SUCCESS')
  3217. self.assertEqual(self.getJobFromHistory('project-test2').result,
  3218. 'SUCCESS')
  3219. self.assertEqual(A.data['status'], 'MERGED')
  3220. self.assertEqual(A.reported, 2)
  3221. # Delete org/new-project zuul repo. Should be recloned.
  3222. p = 'org/delete-project'
  3223. if os.path.exists(os.path.join(self.merger_src_root, p)):
  3224. shutil.rmtree(os.path.join(self.merger_src_root, p))
  3225. if os.path.exists(os.path.join(self.executor_src_root, p)):
  3226. shutil.rmtree(os.path.join(self.executor_src_root, p))
  3227. B = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'B')
  3228. B.addApproval('Code-Review', 2)
  3229. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  3230. self.waitUntilSettled()
  3231. self.assertEqual(self.getJobFromHistory('project-merge').result,
  3232. 'SUCCESS')
  3233. self.assertEqual(self.getJobFromHistory('project-test1').result,
  3234. 'SUCCESS')
  3235. self.assertEqual(self.getJobFromHistory('project-test2').result,
  3236. 'SUCCESS')
  3237. self.assertEqual(B.data['status'], 'MERGED')
  3238. self.assertEqual(B.reported, 2)
  3239. @simple_layout('layouts/untrusted-secrets.yaml')
  3240. def test_untrusted_secrets(self):
  3241. "Test untrusted secrets"
  3242. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  3243. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  3244. self.waitUntilSettled()
  3245. self.assertHistory([])
  3246. self.assertEqual(A.patchsets[0]['approvals'][0]['value'], "-1")
  3247. self.assertIn('does not allow post-review job',
  3248. A.messages[0])
  3249. @simple_layout('layouts/tags.yaml')
  3250. def test_tags(self):
  3251. "Test job tags"
  3252. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  3253. B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
  3254. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  3255. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  3256. self.waitUntilSettled()
  3257. self.assertEqual(len(self.history), 2)
  3258. results = {self.getJobFromHistory('merge',
  3259. project='org/project1').uuid: ['extratag', 'merge'],
  3260. self.getJobFromHistory('merge',
  3261. project='org/project2').uuid: ['merge']}
  3262. for build in self.history:
  3263. self.assertEqual(results.get(build.uuid, ''),
  3264. build.parameters['zuul'].get('jobtags'))
  3265. def test_timer_template(self):
  3266. "Test that a periodic job is triggered"
  3267. # This test can not use simple_layout because it must start
  3268. # with a configuration which does not include a
  3269. # timer-triggered job so that we have an opportunity to set
  3270. # the hold flag before the first job.
  3271. self.create_branch('org/project', 'stable')
  3272. self.executor_server.hold_jobs_in_build = True
  3273. self.commitConfigUpdate('common-config', 'layouts/timer-template.yaml')
  3274. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3275. # The pipeline triggers every second, so we should have seen
  3276. # several by now.
  3277. time.sleep(5)
  3278. self.waitUntilSettled()
  3279. self.assertEqual(len(self.builds), 2)
  3280. merge_count_project1 = 0
  3281. for job in self.gearman_server.jobs_history:
  3282. if job.name == b'merger:refstate':
  3283. args = job.arguments
  3284. if isinstance(args, bytes):
  3285. args = args.decode('utf-8')
  3286. args = json.loads(args)
  3287. if args["items"][0]["project"] == "org/project1":
  3288. merge_count_project1 += 1
  3289. self.assertEquals(merge_count_project1, 0,
  3290. "project1 shouldn't have any refstate call")
  3291. self.executor_server.hold_jobs_in_build = False
  3292. # Stop queuing timer triggered jobs so that the assertions
  3293. # below don't race against more jobs being queued.
  3294. self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
  3295. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3296. self.waitUntilSettled()
  3297. # If APScheduler is in mid-event when we remove the job, we
  3298. # can end up with one more event firing, so give it an extra
  3299. # second to settle.
  3300. time.sleep(1)
  3301. self.waitUntilSettled()
  3302. self.executor_server.release()
  3303. self.waitUntilSettled()
  3304. self.assertHistory([
  3305. dict(name='project-bitrot', result='SUCCESS',
  3306. ref='refs/heads/master'),
  3307. dict(name='project-bitrot', result='SUCCESS',
  3308. ref='refs/heads/stable'),
  3309. ], ordered=False)
  3310. def _test_timer(self, config_file):
  3311. # This test can not use simple_layout because it must start
  3312. # with a configuration which does not include a
  3313. # timer-triggered job so that we have an opportunity to set
  3314. # the hold flag before the first job.
  3315. self.create_branch('org/project', 'stable')
  3316. self.executor_server.hold_jobs_in_build = True
  3317. self.commitConfigUpdate('common-config', config_file)
  3318. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3319. # The pipeline triggers every second, so we should have seen
  3320. # several by now.
  3321. for _ in iterate_timeout(60, 'jobs started'):
  3322. if len(self.builds) > 1:
  3323. break
  3324. merge_count_project1 = 0
  3325. for job in self.gearman_server.jobs_history:
  3326. if job.name == b'merger:refstate':
  3327. args = job.arguments
  3328. if isinstance(args, bytes):
  3329. args = args.decode('utf-8')
  3330. args = json.loads(args)
  3331. if args["items"][0]["project"] == "org/project1":
  3332. merge_count_project1 += 1
  3333. self.assertEquals(merge_count_project1, 0,
  3334. "project1 shouldn't have any refstate call")
  3335. # Ensure that the status json has the ref so we can render it in the
  3336. # web ui.
  3337. data = json.loads(self.scheds.first.sched
  3338. .formatStatusJSON('tenant-one'))
  3339. pipeline = [x for x in data['pipelines'] if x['name'] == 'periodic'][0]
  3340. first = pipeline['change_queues'][0]['heads'][0][0]
  3341. second = pipeline['change_queues'][1]['heads'][0][0]
  3342. self.assertIn(first['ref'], ['refs/heads/master', 'refs/heads/stable'])
  3343. self.assertIn(second['ref'],
  3344. ['refs/heads/master', 'refs/heads/stable'])
  3345. self.executor_server.hold_jobs_in_build = False
  3346. # Stop queuing timer triggered jobs so that the assertions
  3347. # below don't race against more jobs being queued.
  3348. self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
  3349. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3350. self.waitUntilSettled()
  3351. # If APScheduler is in mid-event when we remove the job, we
  3352. # can end up with one more event firing, so give it an extra
  3353. # second to settle.
  3354. time.sleep(3)
  3355. self.waitUntilSettled()
  3356. self.executor_server.release()
  3357. self.waitUntilSettled()
  3358. self.assertTrue(len(self.history) > 1)
  3359. for job in self.history[:1]:
  3360. self.assertEqual(job.result, 'SUCCESS')
  3361. self.assertEqual(job.name, 'project-bitrot')
  3362. self.assertIn(job.ref, ('refs/heads/stable', 'refs/heads/master'))
  3363. def test_timer(self):
  3364. "Test that a periodic job is triggered"
  3365. self._test_timer('layouts/timer.yaml')
  3366. def test_timer_with_jitter(self):
  3367. "Test that a periodic job with a jitter is triggered"
  3368. self._test_timer('layouts/timer-jitter.yaml')
  3369. def test_idle(self):
  3370. "Test that frequent periodic jobs work"
  3371. # This test can not use simple_layout because it must start
  3372. # with a configuration which does not include a
  3373. # timer-triggered job so that we have an opportunity to set
  3374. # the hold flag before the first job.
  3375. self.executor_server.hold_jobs_in_build = True
  3376. for x in range(1, 3):
  3377. # Test that timer triggers periodic jobs even across
  3378. # layout config reloads.
  3379. # Start timer trigger
  3380. self.commitConfigUpdate('common-config',
  3381. 'layouts/idle.yaml')
  3382. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3383. self.waitUntilSettled()
  3384. # The pipeline triggers every second, so we should have seen
  3385. # several by now.
  3386. time.sleep(5)
  3387. # Stop queuing timer triggered jobs so that the assertions
  3388. # below don't race against more jobs being queued.
  3389. self.commitConfigUpdate('common-config',
  3390. 'layouts/no-timer.yaml')
  3391. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3392. self.waitUntilSettled()
  3393. # If APScheduler is in mid-event when we remove the job,
  3394. # we can end up with one more event firing, so give it an
  3395. # extra second to settle.
  3396. time.sleep(1)
  3397. self.waitUntilSettled()
  3398. self.assertEqual(len(self.builds), 1,
  3399. 'Timer builds iteration #%d' % x)
  3400. self.executor_server.release('.*')
  3401. self.waitUntilSettled()
  3402. self.assertEqual(len(self.builds), 0)
  3403. self.assertEqual(len(self.history), x)
  3404. @simple_layout('layouts/smtp.yaml')
  3405. def test_check_smtp_pool(self):
  3406. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3407. self.waitUntilSettled()
  3408. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  3409. self.waitUntilSettled()
  3410. self.assertEqual(len(self.smtp_messages), 2)
  3411. # A.messages only holds what FakeGerrit places in it. Thus we
  3412. # work on the knowledge of what the first message should be as
  3413. # it is only configured to go to SMTP.
  3414. self.assertEqual('zuul@example.com',
  3415. self.smtp_messages[0]['from_email'])
  3416. self.assertEqual(['you@example.com'],
  3417. self.smtp_messages[0]['to_email'])
  3418. self.assertEqual('Starting check jobs.',
  3419. self.smtp_messages[0]['body'])
  3420. self.assertEqual('zuul_from@example.com',
  3421. self.smtp_messages[1]['from_email'])
  3422. self.assertEqual(['alternative_me@example.com'],
  3423. self.smtp_messages[1]['to_email'])
  3424. self.assertEqual(A.messages[0],
  3425. self.smtp_messages[1]['body'])
  3426. @simple_layout('layouts/smtp.yaml')
  3427. @mock.patch('zuul.driver.gerrit.gerritreporter.GerritReporter.report')
  3428. def test_failed_reporter(self, report_mock):
  3429. '''Test that one failed reporter doesn't break other reporters'''
  3430. # Warning hacks. We sort the reports here so that the test is
  3431. # deterministic. Gerrit reporting will fail, but smtp reporting
  3432. # should succeed.
  3433. report_mock.side_effect = Exception('Gerrit failed to report')
  3434. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  3435. check = tenant.layout.pipelines['check']
  3436. check.success_actions = sorted(check.success_actions,
  3437. key=lambda x: x.name)
  3438. self.assertEqual(check.success_actions[0].name, 'gerrit')
  3439. self.assertEqual(check.success_actions[1].name, 'smtp')
  3440. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3441. self.waitUntilSettled()
  3442. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  3443. self.waitUntilSettled()
  3444. # We know that if gerrit ran first and failed and smtp ran second
  3445. # and sends mail then we handle failures in reporters gracefully.
  3446. self.assertEqual(len(self.smtp_messages), 2)
  3447. # A.messages only holds what FakeGerrit places in it. Thus we
  3448. # work on the knowledge of what the first message should be as
  3449. # it is only configured to go to SMTP.
  3450. self.assertEqual('zuul@example.com',
  3451. self.smtp_messages[0]['from_email'])
  3452. self.assertEqual(['you@example.com'],
  3453. self.smtp_messages[0]['to_email'])
  3454. self.assertEqual('Starting check jobs.',
  3455. self.smtp_messages[0]['body'])
  3456. self.assertEqual('zuul_from@example.com',
  3457. self.smtp_messages[1]['from_email'])
  3458. self.assertEqual(['alternative_me@example.com'],
  3459. self.smtp_messages[1]['to_email'])
  3460. # This double checks that Gerrit side failed
  3461. self.assertEqual(A.messages, [])
  3462. def test_timer_smtp(self):
  3463. "Test that a periodic job is triggered"
  3464. # This test can not use simple_layout because it must start
  3465. # with a configuration which does not include a
  3466. # timer-triggered job so that we have an opportunity to set
  3467. # the hold flag before the first job.
  3468. self.executor_server.hold_jobs_in_build = True
  3469. self.commitConfigUpdate('common-config', 'layouts/timer-smtp.yaml')
  3470. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3471. # The pipeline triggers every second, so we should have seen
  3472. # several by now.
  3473. time.sleep(5)
  3474. self.waitUntilSettled()
  3475. self.assertEqual(len(self.builds), 2)
  3476. self.executor_server.release('.*')
  3477. self.waitUntilSettled()
  3478. self.assertEqual(len(self.history), 2)
  3479. self.assertEqual(self.getJobFromHistory(
  3480. 'project-bitrot-stable-old').result, 'SUCCESS')
  3481. self.assertEqual(self.getJobFromHistory(
  3482. 'project-bitrot-stable-older').result, 'SUCCESS')
  3483. self.assertEqual(len(self.smtp_messages), 1)
  3484. # A.messages only holds what FakeGerrit places in it. Thus we
  3485. # work on the knowledge of what the first message should be as
  3486. # it is only configured to go to SMTP.
  3487. self.assertEqual('zuul_from@example.com',
  3488. self.smtp_messages[0]['from_email'])
  3489. self.assertEqual(['alternative_me@example.com'],
  3490. self.smtp_messages[0]['to_email'])
  3491. self.assertIn('Subject: Periodic check for org/project succeeded',
  3492. self.smtp_messages[0]['headers'])
  3493. # Stop queuing timer triggered jobs and let any that may have
  3494. # queued through so that end of test assertions pass.
  3495. self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
  3496. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3497. self.waitUntilSettled()
  3498. # If APScheduler is in mid-event when we remove the job, we
  3499. # can end up with one more event firing, so give it an extra
  3500. # second to settle.
  3501. time.sleep(1)
  3502. self.waitUntilSettled()
  3503. self.executor_server.release('.*')
  3504. self.waitUntilSettled()
  3505. @skip("Disabled for early v3 development")
  3506. def test_timer_sshkey(self):
  3507. "Test that a periodic job can setup SSH key authentication"
  3508. self.worker.hold_jobs_in_build = True
  3509. self.config.set('zuul', 'layout_config',
  3510. 'tests/fixtures/layout-timer.yaml')
  3511. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3512. self.registerJobs()
  3513. # The pipeline triggers every second, so we should have seen
  3514. # several by now.
  3515. time.sleep(5)
  3516. self.waitUntilSettled()
  3517. self.assertEqual(len(self.builds), 2)
  3518. ssh_wrapper = os.path.join(self.git_root, ".ssh_wrapper_gerrit")
  3519. self.assertTrue(os.path.isfile(ssh_wrapper))
  3520. with open(ssh_wrapper) as f:
  3521. ssh_wrapper_content = f.read()
  3522. self.assertIn("fake_id_rsa", ssh_wrapper_content)
  3523. # In the unit tests Merger runs in the same process,
  3524. # so we see its' environment variables
  3525. self.assertEqual(os.environ['GIT_SSH'], ssh_wrapper)
  3526. self.worker.release('.*')
  3527. self.waitUntilSettled()
  3528. self.assertEqual(len(self.history), 2)
  3529. self.assertEqual(self.getJobFromHistory(
  3530. 'project-bitrot-stable-old').result, 'SUCCESS')
  3531. self.assertEqual(self.getJobFromHistory(
  3532. 'project-bitrot-stable-older').result, 'SUCCESS')
  3533. # Stop queuing timer triggered jobs and let any that may have
  3534. # queued through so that end of test assertions pass.
  3535. self.config.set('zuul', 'layout_config',
  3536. 'tests/fixtures/layout-no-timer.yaml')
  3537. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3538. self.registerJobs()
  3539. self.waitUntilSettled()
  3540. # If APScheduler is in mid-event when we remove the job, we
  3541. # can end up with one more event firing, so give it an extra
  3542. # second to settle.
  3543. time.sleep(1)
  3544. self.waitUntilSettled()
  3545. self.worker.release('.*')
  3546. self.waitUntilSettled()
  3547. def test_client_enqueue_change_with_trigger(self):
  3548. """Test that the RPC client can enqueue a change with the deprecated
  3549. trigger argument"""
  3550. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3551. A.addApproval('Code-Review', 2)
  3552. A.addApproval('Approved', 1)
  3553. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3554. self.gearman_server.port)
  3555. self.addCleanup(client.shutdown)
  3556. r = client.enqueue(tenant='tenant-one',
  3557. pipeline='gate',
  3558. project='org/project',
  3559. trigger='gerrit',
  3560. change='1,1')
  3561. self.waitUntilSettled()
  3562. self.assertEqual(self.getJobFromHistory('project-merge').result,
  3563. 'SUCCESS')
  3564. self.assertEqual(self.getJobFromHistory('project-test1').result,
  3565. 'SUCCESS')
  3566. self.assertEqual(self.getJobFromHistory('project-test2').result,
  3567. 'SUCCESS')
  3568. self.assertEqual(A.data['status'], 'MERGED')
  3569. self.assertEqual(A.reported, 2)
  3570. self.assertEqual(r, True)
  3571. def test_client_enqueue_change_no_trigger(self):
  3572. """Test that the RPC client can enqueue a change without an explicit
  3573. trigger"""
  3574. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3575. A.addApproval('Code-Review', 2)
  3576. A.addApproval('Approved', 1)
  3577. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3578. self.gearman_server.port)
  3579. self.addCleanup(client.shutdown)
  3580. r = client.enqueue(tenant='tenant-one',
  3581. pipeline='gate',
  3582. project='org/project',
  3583. trigger=None,
  3584. change='1,1')
  3585. self.waitUntilSettled()
  3586. self.assertEqual(self.getJobFromHistory('project-merge').result,
  3587. 'SUCCESS')
  3588. self.assertEqual(self.getJobFromHistory('project-test1').result,
  3589. 'SUCCESS')
  3590. self.assertEqual(self.getJobFromHistory('project-test2').result,
  3591. 'SUCCESS')
  3592. self.assertEqual(A.data['status'], 'MERGED')
  3593. self.assertEqual(A.reported, 2)
  3594. self.assertEqual(r, True)
  3595. @simple_layout('layouts/three-projects.yaml')
  3596. def test_client_enqueue_change_wrong_project(self):
  3597. "Test that an enqueue fails if a change doesn't belong to the project"
  3598. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  3599. A.addApproval('Code-Review', 2)
  3600. A.addApproval('Approved', 1)
  3601. B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
  3602. B.addApproval('Code-Review', 2)
  3603. B.addApproval('Approved', 1)
  3604. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3605. self.gearman_server.port)
  3606. self.addCleanup(client.shutdown)
  3607. with testtools.ExpectedException(
  3608. zuul.rpcclient.RPCFailure,
  3609. 'Change 2,1 does not belong to project "org/project1"'):
  3610. r = client.enqueue(tenant='tenant-one',
  3611. pipeline='gate',
  3612. project='org/project1',
  3613. trigger=None,
  3614. change='2,1')
  3615. self.assertEqual(r, False)
  3616. self.waitUntilSettled()
  3617. def test_client_enqueue_ref(self):
  3618. "Test that the RPC client can enqueue a ref"
  3619. p = "review.example.com/org/project"
  3620. upstream = self.getUpstreamRepos([p])
  3621. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3622. A.setMerged()
  3623. A_commit = str(upstream[p].commit('master'))
  3624. self.log.debug("A commit: %s" % A_commit)
  3625. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3626. self.gearman_server.port)
  3627. self.addCleanup(client.shutdown)
  3628. r = client.enqueue_ref(
  3629. tenant='tenant-one',
  3630. pipeline='post',
  3631. project='org/project',
  3632. trigger=None,
  3633. ref='master',
  3634. oldrev='90f173846e3af9154517b88543ffbd1691f31366',
  3635. newrev=A_commit)
  3636. self.waitUntilSettled()
  3637. job_names = [x.name for x in self.history]
  3638. self.assertEqual(len(self.history), 1)
  3639. self.assertIn('project-post', job_names)
  3640. self.assertEqual(r, True)
  3641. def test_client_dequeue_ref(self):
  3642. "Test that the RPC client can dequeue a ref"
  3643. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3644. self.gearman_server.port)
  3645. self.addCleanup(client.shutdown)
  3646. self.executor_server.hold_jobs_in_build = True
  3647. p = "review.example.com/org/project"
  3648. upstream = self.getUpstreamRepos([p])
  3649. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3650. A.setMerged()
  3651. A_commit = str(upstream[p].commit('master'))
  3652. self.log.debug("A commit: %s" % A_commit)
  3653. r = client.enqueue_ref(
  3654. tenant='tenant-one',
  3655. pipeline='post',
  3656. project='org/project',
  3657. trigger=None,
  3658. ref='master',
  3659. oldrev='90f173846e3af9154517b88543ffbd1691f31366',
  3660. newrev=A_commit)
  3661. p = "review.example.com/org/project1"
  3662. upstream = self.getUpstreamRepos([p])
  3663. B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
  3664. B.setMerged()
  3665. B_commit = str(upstream[p].commit('master'))
  3666. self.log.debug("B commit: %s" % B_commit)
  3667. r = client.enqueue_ref(
  3668. tenant='tenant-one',
  3669. pipeline='post',
  3670. project='org/project1',
  3671. trigger=None,
  3672. ref='master',
  3673. oldrev='90f173846e3af9154517b88543ffbd1691f31366',
  3674. newrev=B_commit)
  3675. self.waitUntilSettled()
  3676. r = client.dequeue(
  3677. tenant='tenant-one',
  3678. pipeline='post',
  3679. project='org/project1',
  3680. change=None,
  3681. ref='master')
  3682. self.waitUntilSettled()
  3683. self.executor_server.release('.*')
  3684. self.waitUntilSettled()
  3685. job_names = [x.name for x in self.history]
  3686. self.assertEqual(len(self.history), 2)
  3687. aborted_build = [
  3688. build for build in self.history if
  3689. build.result == 'ABORTED'][0]
  3690. self.assertEqual(aborted_build.newrev, B_commit)
  3691. self.assertIn('project-post', job_names)
  3692. self.assertEqual(r, True)
  3693. def test_client_dequeue_dependent_change(self):
  3694. "Test that the RPC client can dequeue a change"
  3695. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3696. self.gearman_server.port)
  3697. self.addCleanup(client.shutdown)
  3698. self.executor_server.hold_jobs_in_build = True
  3699. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3700. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  3701. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  3702. C.setDependsOn(B, 1)
  3703. B.setDependsOn(A, 1)
  3704. A.addApproval('Code-Review', 2)
  3705. B.addApproval('Code-Review', 2)
  3706. C.addApproval('Code-Review', 2)
  3707. # Promote to 'gate' pipeline
  3708. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  3709. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  3710. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  3711. self.waitUntilSettled()
  3712. client.dequeue(
  3713. tenant='tenant-one',
  3714. pipeline='gate',
  3715. project='org/project',
  3716. change='1,1',
  3717. ref=None)
  3718. self.waitUntilSettled()
  3719. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  3720. gate_pipeline = tenant.layout.pipelines['gate']
  3721. self.assertEqual(gate_pipeline.getAllItems(), [])
  3722. self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1)
  3723. self.executor_server.hold_jobs_in_build = False
  3724. self.executor_server.release()
  3725. self.waitUntilSettled()
  3726. def test_client_dequeue_independent_change(self):
  3727. "Test that the RPC client can dequeue a change"
  3728. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3729. self.gearman_server.port)
  3730. self.addCleanup(client.shutdown)
  3731. self.executor_server.hold_jobs_in_build = True
  3732. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3733. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  3734. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  3735. A.addApproval('Code-Review', 2)
  3736. B.addApproval('Code-Review', 2)
  3737. C.addApproval('Code-Review', 2)
  3738. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  3739. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  3740. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  3741. self.waitUntilSettled()
  3742. client.dequeue(
  3743. tenant='tenant-one',
  3744. pipeline='check',
  3745. project='org/project',
  3746. change='1,1',
  3747. ref=None)
  3748. self.waitUntilSettled()
  3749. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  3750. check_pipeline = tenant.layout.pipelines['check']
  3751. self.assertEqual(len(check_pipeline.getAllItems()), 2)
  3752. self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1)
  3753. self.executor_server.hold_jobs_in_build = False
  3754. self.executor_server.release()
  3755. self.waitUntilSettled()
  3756. @simple_layout('layouts/three-projects.yaml')
  3757. def test_client_dequeue_wrong_project(self):
  3758. "Test that dequeue fails if change and project do not match"
  3759. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3760. self.gearman_server.port)
  3761. self.addCleanup(client.shutdown)
  3762. self.executor_server.hold_jobs_in_build = True
  3763. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  3764. B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
  3765. C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')
  3766. D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
  3767. A.addApproval('Code-Review', 2)
  3768. B.addApproval('Code-Review', 2)
  3769. C.addApproval('Code-Review', 2)
  3770. D.addApproval('Code-Review', 2)
  3771. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  3772. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  3773. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  3774. self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
  3775. self.waitUntilSettled()
  3776. with testtools.ExpectedException(
  3777. zuul.rpcclient.RPCFailure,
  3778. 'Change 4,1 does not belong to project "org/project1"'):
  3779. r = client.dequeue(
  3780. tenant='tenant-one',
  3781. pipeline='check',
  3782. project='org/project1',
  3783. change='4,1',
  3784. ref=None)
  3785. self.waitUntilSettled()
  3786. self.assertEqual(r, False)
  3787. self.executor_server.hold_jobs_in_build = False
  3788. self.executor_server.release()
  3789. self.waitUntilSettled()
  3790. def test_client_dequeue_change_by_ref(self):
  3791. "Test that the RPC client can dequeue a change by ref"
  3792. # Test this on the periodic pipeline, where it makes most sense to
  3793. # use ref
  3794. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3795. self.gearman_server.port)
  3796. self.addCleanup(client.shutdown)
  3797. self.create_branch('org/project', 'stable')
  3798. self.fake_gerrit.addEvent(
  3799. self.fake_gerrit.getFakeBranchCreatedEvent(
  3800. 'org/project', 'stable'))
  3801. self.waitUntilSettled()
  3802. self.executor_server.hold_jobs_in_build = True
  3803. self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
  3804. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3805. # We expect that one build for each branch (master and stable) appears.
  3806. for _ in iterate_timeout(30, 'Wait for two builds that are hold'):
  3807. if len(self.builds) == 2:
  3808. break
  3809. self.waitUntilSettled()
  3810. client.dequeue(
  3811. tenant='tenant-one',
  3812. pipeline='periodic',
  3813. project='org/project',
  3814. change=None,
  3815. ref='refs/heads/stable')
  3816. self.waitUntilSettled()
  3817. self.commitConfigUpdate('common-config',
  3818. 'layouts/no-timer.yaml')
  3819. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  3820. self.waitUntilSettled()
  3821. self.executor_server.hold_jobs_in_build = False
  3822. self.executor_server.release()
  3823. self.waitUntilSettled()
  3824. self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1)
  3825. def test_client_enqueue_negative(self):
  3826. "Test that the RPC client returns errors"
  3827. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3828. self.gearman_server.port)
  3829. self.addCleanup(client.shutdown)
  3830. with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
  3831. "Invalid tenant"):
  3832. r = client.enqueue(tenant='tenant-foo',
  3833. pipeline='gate',
  3834. project='org/project',
  3835. trigger=None,
  3836. change='1,1')
  3837. self.assertEqual(r, False)
  3838. with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
  3839. "Invalid project"):
  3840. r = client.enqueue(tenant='tenant-one',
  3841. pipeline='gate',
  3842. project='project-does-not-exist',
  3843. trigger=None,
  3844. change='1,1')
  3845. self.assertEqual(r, False)
  3846. with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
  3847. "Invalid pipeline"):
  3848. r = client.enqueue(tenant='tenant-one',
  3849. pipeline='pipeline-does-not-exist',
  3850. project='org/project',
  3851. trigger=None,
  3852. change='1,1')
  3853. self.assertEqual(r, False)
  3854. with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
  3855. "Invalid change"):
  3856. r = client.enqueue(tenant='tenant-one',
  3857. pipeline='gate',
  3858. project='org/project',
  3859. trigger=None,
  3860. change='1,1')
  3861. self.assertEqual(r, False)
  3862. self.waitUntilSettled()
  3863. self.assertEqual(len(self.history), 0)
  3864. self.assertEqual(len(self.builds), 0)
  3865. def test_client_enqueue_ref_negative(self):
  3866. "Test that the RPC client returns errors"
  3867. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3868. self.gearman_server.port)
  3869. self.addCleanup(client.shutdown)
  3870. with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
  3871. "New rev must be 40 character sha1"):
  3872. r = client.enqueue_ref(
  3873. tenant='tenant-one',
  3874. pipeline='post',
  3875. project='org/project',
  3876. trigger=None,
  3877. ref='master',
  3878. oldrev='90f173846e3af9154517b88543ffbd1691f31366',
  3879. newrev='10054041')
  3880. self.assertEqual(r, False)
  3881. with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
  3882. "Old rev must be 40 character sha1"):
  3883. r = client.enqueue_ref(
  3884. tenant='tenant-one',
  3885. pipeline='post',
  3886. project='org/project',
  3887. trigger=None,
  3888. ref='master',
  3889. oldrev='10054041',
  3890. newrev='90f173846e3af9154517b88543ffbd1691f31366')
  3891. self.assertEqual(r, False)
  3892. with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
  3893. "New rev must be base16 hash"):
  3894. r = client.enqueue_ref(
  3895. tenant='tenant-one',
  3896. pipeline='post',
  3897. project='org/project',
  3898. trigger=None,
  3899. ref='master',
  3900. oldrev='90f173846e3af9154517b88543ffbd1691f31366',
  3901. newrev='notbase16')
  3902. self.assertEqual(r, False)
  3903. with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
  3904. "Old rev must be base16 hash"):
  3905. r = client.enqueue_ref(
  3906. tenant='tenant-one',
  3907. pipeline='post',
  3908. project='org/project',
  3909. trigger=None,
  3910. ref='master',
  3911. oldrev='notbase16',
  3912. newrev='90f173846e3af9154517b88543ffbd1691f31366')
  3913. self.assertEqual(r, False)
  3914. def test_client_promote(self):
  3915. "Test that the RPC client can promote a change"
  3916. self.executor_server.hold_jobs_in_build = True
  3917. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3918. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  3919. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  3920. A.addApproval('Code-Review', 2)
  3921. B.addApproval('Code-Review', 2)
  3922. C.addApproval('Code-Review', 2)
  3923. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  3924. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  3925. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  3926. self.waitUntilSettled()
  3927. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  3928. items = tenant.layout.pipelines['gate'].getAllItems()
  3929. enqueue_times = {}
  3930. for item in items:
  3931. enqueue_times[str(item.change)] = item.enqueue_time
  3932. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3933. self.gearman_server.port)
  3934. self.addCleanup(client.shutdown)
  3935. r = client.promote(tenant='tenant-one',
  3936. pipeline='gate',
  3937. change_ids=['2,1', '3,1'])
  3938. # ensure that enqueue times are durable
  3939. items = tenant.layout.pipelines['gate'].getAllItems()
  3940. for item in items:
  3941. self.assertEqual(
  3942. enqueue_times[str(item.change)], item.enqueue_time)
  3943. self.waitUntilSettled()
  3944. self.executor_server.release('.*-merge')
  3945. self.waitUntilSettled()
  3946. self.executor_server.release('.*-merge')
  3947. self.waitUntilSettled()
  3948. self.executor_server.release('.*-merge')
  3949. self.waitUntilSettled()
  3950. self.assertEqual(len(self.builds), 6)
  3951. self.assertEqual(self.builds[0].name, 'project-test1')
  3952. self.assertEqual(self.builds[1].name, 'project-test2')
  3953. self.assertEqual(self.builds[2].name, 'project-test1')
  3954. self.assertEqual(self.builds[3].name, 'project-test2')
  3955. self.assertEqual(self.builds[4].name, 'project-test1')
  3956. self.assertEqual(self.builds[5].name, 'project-test2')
  3957. self.assertTrue(self.builds[0].hasChanges(B))
  3958. self.assertFalse(self.builds[0].hasChanges(A))
  3959. self.assertFalse(self.builds[0].hasChanges(C))
  3960. self.assertTrue(self.builds[2].hasChanges(B))
  3961. self.assertTrue(self.builds[2].hasChanges(C))
  3962. self.assertFalse(self.builds[2].hasChanges(A))
  3963. self.assertTrue(self.builds[4].hasChanges(B))
  3964. self.assertTrue(self.builds[4].hasChanges(C))
  3965. self.assertTrue(self.builds[4].hasChanges(A))
  3966. self.executor_server.release()
  3967. self.waitUntilSettled()
  3968. self.assertEqual(A.data['status'], 'MERGED')
  3969. self.assertEqual(A.reported, 2)
  3970. self.assertEqual(B.data['status'], 'MERGED')
  3971. self.assertEqual(B.reported, 2)
  3972. self.assertEqual(C.data['status'], 'MERGED')
  3973. self.assertEqual(C.reported, 2)
  3974. self.assertEqual(r, True)
  3975. def test_client_promote_dependent(self):
  3976. "Test that the RPC client can promote a dependent change"
  3977. # C (depends on B) -> B -> A ; then promote C to get:
  3978. # A -> C (depends on B) -> B
  3979. self.executor_server.hold_jobs_in_build = True
  3980. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  3981. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  3982. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  3983. C.setDependsOn(B, 1)
  3984. A.addApproval('Code-Review', 2)
  3985. B.addApproval('Code-Review', 2)
  3986. C.addApproval('Code-Review', 2)
  3987. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  3988. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  3989. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  3990. self.waitUntilSettled()
  3991. client = zuul.rpcclient.RPCClient('127.0.0.1',
  3992. self.gearman_server.port)
  3993. self.addCleanup(client.shutdown)
  3994. r = client.promote(tenant='tenant-one',
  3995. pipeline='gate',
  3996. change_ids=['3,1'])
  3997. self.waitUntilSettled()
  3998. self.executor_server.release('.*-merge')
  3999. self.waitUntilSettled()
  4000. self.executor_server.release('.*-merge')
  4001. self.waitUntilSettled()
  4002. self.executor_server.release('.*-merge')
  4003. self.waitUntilSettled()
  4004. self.assertEqual(len(self.builds), 6)
  4005. self.assertEqual(self.builds[0].name, 'project-test1')
  4006. self.assertEqual(self.builds[1].name, 'project-test2')
  4007. self.assertEqual(self.builds[2].name, 'project-test1')
  4008. self.assertEqual(self.builds[3].name, 'project-test2')
  4009. self.assertEqual(self.builds[4].name, 'project-test1')
  4010. self.assertEqual(self.builds[5].name, 'project-test2')
  4011. self.assertTrue(self.builds[0].hasChanges(B))
  4012. self.assertFalse(self.builds[0].hasChanges(A))
  4013. self.assertFalse(self.builds[0].hasChanges(C))
  4014. self.assertTrue(self.builds[2].hasChanges(B))
  4015. self.assertTrue(self.builds[2].hasChanges(C))
  4016. self.assertFalse(self.builds[2].hasChanges(A))
  4017. self.assertTrue(self.builds[4].hasChanges(B))
  4018. self.assertTrue(self.builds[4].hasChanges(C))
  4019. self.assertTrue(self.builds[4].hasChanges(A))
  4020. self.executor_server.release()
  4021. self.waitUntilSettled()
  4022. self.assertEqual(A.data['status'], 'MERGED')
  4023. self.assertEqual(A.reported, 2)
  4024. self.assertEqual(B.data['status'], 'MERGED')
  4025. self.assertEqual(B.reported, 2)
  4026. self.assertEqual(C.data['status'], 'MERGED')
  4027. self.assertEqual(C.reported, 2)
  4028. self.assertEqual(r, True)
  4029. def test_client_promote_negative(self):
  4030. "Test that the RPC client returns errors for promotion"
  4031. self.executor_server.hold_jobs_in_build = True
  4032. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4033. A.addApproval('Code-Review', 2)
  4034. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4035. self.waitUntilSettled()
  4036. client = zuul.rpcclient.RPCClient('127.0.0.1',
  4037. self.gearman_server.port)
  4038. self.addCleanup(client.shutdown)
  4039. with testtools.ExpectedException(zuul.rpcclient.RPCFailure):
  4040. r = client.promote(tenant='tenant-one',
  4041. pipeline='nonexistent',
  4042. change_ids=['2,1', '3,1'])
  4043. self.assertEqual(r, False)
  4044. with testtools.ExpectedException(zuul.rpcclient.RPCFailure):
  4045. r = client.promote(tenant='tenant-one',
  4046. pipeline='gate',
  4047. change_ids=['4,1'])
  4048. self.assertEqual(r, False)
  4049. self.executor_server.hold_jobs_in_build = False
  4050. self.executor_server.release()
  4051. self.waitUntilSettled()
  4052. @simple_layout('layouts/rate-limit.yaml')
  4053. def test_queue_rate_limiting(self):
  4054. "Test that DependentPipelines are rate limited with dep across window"
  4055. self.executor_server.hold_jobs_in_build = True
  4056. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4057. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  4058. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  4059. C.setDependsOn(B, 1)
  4060. self.executor_server.failJob('project-test1', A)
  4061. A.addApproval('Code-Review', 2)
  4062. B.addApproval('Code-Review', 2)
  4063. C.addApproval('Code-Review', 2)
  4064. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4065. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  4066. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  4067. self.waitUntilSettled()
  4068. # Only A and B will have their merge jobs queued because
  4069. # window is 2.
  4070. self.assertEqual(len(self.builds), 2)
  4071. self.assertEqual(self.builds[0].name, 'project-merge')
  4072. self.assertEqual(self.builds[1].name, 'project-merge')
  4073. # Release the merge jobs one at a time.
  4074. self.builds[0].release()
  4075. self.waitUntilSettled()
  4076. self.builds[0].release()
  4077. self.waitUntilSettled()
  4078. # Only A and B will have their test jobs queued because
  4079. # window is 2.
  4080. self.assertEqual(len(self.builds), 4)
  4081. self.assertEqual(self.builds[0].name, 'project-test1')
  4082. self.assertEqual(self.builds[1].name, 'project-test2')
  4083. self.assertEqual(self.builds[2].name, 'project-test1')
  4084. self.assertEqual(self.builds[3].name, 'project-test2')
  4085. self.executor_server.release('project-.*')
  4086. self.waitUntilSettled()
  4087. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4088. queue = tenant.layout.pipelines['gate'].queues[0]
  4089. # A failed so window is reduced by 1 to 1.
  4090. self.assertEqual(queue.window, 1)
  4091. self.assertEqual(queue.window_floor, 1)
  4092. self.assertEqual(A.data['status'], 'NEW')
  4093. # Gate is reset and only B's merge job is queued because
  4094. # window shrunk to 1.
  4095. self.assertEqual(len(self.builds), 1)
  4096. self.assertEqual(self.builds[0].name, 'project-merge')
  4097. self.executor_server.release('.*-merge')
  4098. self.waitUntilSettled()
  4099. # Only B's test jobs are queued because window is still 1.
  4100. self.assertEqual(len(self.builds), 2)
  4101. self.assertEqual(self.builds[0].name, 'project-test1')
  4102. self.assertEqual(self.builds[1].name, 'project-test2')
  4103. self.executor_server.release('project-.*')
  4104. self.waitUntilSettled()
  4105. # B was successfully merged so window is increased to 2.
  4106. self.assertEqual(queue.window, 2)
  4107. self.assertEqual(queue.window_floor, 1)
  4108. self.assertEqual(B.data['status'], 'MERGED')
  4109. # Only C is left and its merge job is queued.
  4110. self.assertEqual(len(self.builds), 1)
  4111. self.assertEqual(self.builds[0].name, 'project-merge')
  4112. self.executor_server.release('.*-merge')
  4113. self.waitUntilSettled()
  4114. # After successful merge job the test jobs for C are queued.
  4115. self.assertEqual(len(self.builds), 2)
  4116. self.assertEqual(self.builds[0].name, 'project-test1')
  4117. self.assertEqual(self.builds[1].name, 'project-test2')
  4118. self.executor_server.release('project-.*')
  4119. self.waitUntilSettled()
  4120. # C successfully merged so window is bumped to 3.
  4121. self.assertEqual(queue.window, 3)
  4122. self.assertEqual(queue.window_floor, 1)
  4123. self.assertEqual(C.data['status'], 'MERGED')
  4124. @simple_layout('layouts/rate-limit.yaml')
  4125. def test_queue_rate_limiting_dependent(self):
  4126. "Test that DependentPipelines are rate limited with dep in window"
  4127. self.executor_server.hold_jobs_in_build = True
  4128. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4129. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  4130. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  4131. B.setDependsOn(A, 1)
  4132. self.executor_server.failJob('project-test1', A)
  4133. A.addApproval('Code-Review', 2)
  4134. B.addApproval('Code-Review', 2)
  4135. C.addApproval('Code-Review', 2)
  4136. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4137. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  4138. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  4139. self.waitUntilSettled()
  4140. # Only A and B will have their merge jobs queued because
  4141. # window is 2.
  4142. self.assertEqual(len(self.builds), 2)
  4143. self.assertEqual(self.builds[0].name, 'project-merge')
  4144. self.assertEqual(self.builds[1].name, 'project-merge')
  4145. self.orderedRelease(2)
  4146. # Only A and B will have their test jobs queued because
  4147. # window is 2.
  4148. self.assertEqual(len(self.builds), 4)
  4149. self.assertEqual(self.builds[0].name, 'project-test1')
  4150. self.assertEqual(self.builds[1].name, 'project-test2')
  4151. self.assertEqual(self.builds[2].name, 'project-test1')
  4152. self.assertEqual(self.builds[3].name, 'project-test2')
  4153. self.executor_server.release('project-.*')
  4154. self.waitUntilSettled()
  4155. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4156. queue = tenant.layout.pipelines['gate'].queues[0]
  4157. # A failed so window is reduced by 1 to 1.
  4158. self.assertEqual(queue.window, 1)
  4159. self.assertEqual(queue.window_floor, 1)
  4160. self.assertEqual(A.data['status'], 'NEW')
  4161. self.assertEqual(B.data['status'], 'NEW')
  4162. # Gate is reset and only C's merge job is queued because
  4163. # window shrunk to 1 and A and B were dequeued.
  4164. self.assertEqual(len(self.builds), 1)
  4165. self.assertEqual(self.builds[0].name, 'project-merge')
  4166. self.orderedRelease(1)
  4167. # Only C's test jobs are queued because window is still 1.
  4168. self.assertEqual(len(self.builds), 2)
  4169. builds = self.getSortedBuilds()
  4170. self.assertEqual(builds[0].name, 'project-test1')
  4171. self.assertEqual(builds[1].name, 'project-test2')
  4172. self.executor_server.release('project-.*')
  4173. self.waitUntilSettled()
  4174. # C was successfully merged so window is increased to 2.
  4175. self.assertEqual(queue.window, 2)
  4176. self.assertEqual(queue.window_floor, 1)
  4177. self.assertEqual(C.data['status'], 'MERGED')
  4178. @simple_layout('layouts/rate-limit-reconfigure.yaml')
  4179. def test_queue_rate_limiting_reconfigure(self):
  4180. """Test that changes survive a reconfigure when no longer in window.
  4181. This is a regression tests for a case that lead to an exception during
  4182. re-enqueue. The exception happened when former active items had already
  4183. build results but then dropped out of the active window. During
  4184. re-enqueue the job graph was not re-initialized because the items were
  4185. no longer active.
  4186. """
  4187. self.executor_server.hold_jobs_in_build = True
  4188. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4189. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  4190. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  4191. D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
  4192. A.addApproval('Code-Review', 2)
  4193. B.addApproval('Code-Review', 2)
  4194. C.addApproval('Code-Review', 2)
  4195. D.addApproval('Code-Review', 2)
  4196. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4197. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  4198. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  4199. self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
  4200. self.waitUntilSettled()
  4201. self.assertEqual(len(self.builds), 4)
  4202. self.assertEqual(self.builds[0].name, 'project-merge')
  4203. self.assertEqual(self.builds[1].name, 'project-merge')
  4204. self.assertEqual(self.builds[2].name, 'project-merge')
  4205. self.assertEqual(self.builds[3].name, 'project-merge')
  4206. self.orderedRelease(4)
  4207. self.assertEqual(len(self.builds), 8)
  4208. self.assertEqual(self.builds[0].name, 'project-test1')
  4209. self.assertEqual(self.builds[1].name, 'project-test2')
  4210. self.assertEqual(self.builds[2].name, 'project-test1')
  4211. self.assertEqual(self.builds[3].name, 'project-test2')
  4212. self.assertEqual(self.builds[4].name, 'project-test1')
  4213. self.assertEqual(self.builds[5].name, 'project-test2')
  4214. self.assertEqual(self.builds[6].name, 'project-test1')
  4215. self.assertEqual(self.builds[7].name, 'project-test2')
  4216. self.executor_server.failJob('project-test1', B)
  4217. self.builds[2].release()
  4218. self.builds[3].release()
  4219. self.waitUntilSettled()
  4220. self.assertEqual(len(self.builds), 4)
  4221. # A's jobs
  4222. self.assertEqual(self.builds[0].name, 'project-test1')
  4223. self.assertEqual(self.builds[1].name, 'project-test2')
  4224. # C's and D's merge jobs
  4225. self.assertEqual(self.builds[2].name, 'project-merge')
  4226. self.assertEqual(self.builds[3].name, 'project-merge')
  4227. # Release merge jobs of C, D after speculative gate reset
  4228. self.executor_server.release('project-merge')
  4229. self.waitUntilSettled()
  4230. self.assertEqual(len(self.builds), 6)
  4231. # A's jobs
  4232. self.assertEqual(self.builds[0].name, 'project-test1')
  4233. self.assertEqual(self.builds[1].name, 'project-test2')
  4234. # C's + D's jobs
  4235. self.assertEqual(self.builds[2].name, 'project-test1')
  4236. self.assertEqual(self.builds[3].name, 'project-test2')
  4237. self.assertEqual(self.builds[4].name, 'project-test1')
  4238. self.assertEqual(self.builds[5].name, 'project-test2')
  4239. # Fail D's job so we have a build results for an item that
  4240. # is not in the active window after B is reported
  4241. # (condition that previously lead to an exception)
  4242. self.executor_server.failJob('project-test1', D)
  4243. self.builds[4].release()
  4244. self.waitUntilSettled()
  4245. # Release A's jobs
  4246. self.builds[0].release()
  4247. self.builds[1].release()
  4248. self.waitUntilSettled()
  4249. self.assertEqual(len(self.builds), 3)
  4250. # C's jobs
  4251. self.assertEqual(self.builds[0].name, 'project-test1')
  4252. self.assertEqual(self.builds[1].name, 'project-test2')
  4253. # D's remaining job
  4254. self.assertEqual(self.builds[2].name, 'project-test2')
  4255. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4256. queue = tenant.layout.pipelines['gate'].queues[0]
  4257. self.assertEqual(queue.window, 1)
  4258. # D dropped out of the window
  4259. self.assertFalse(queue.queue[-1].active)
  4260. self.commitConfigUpdate('org/common-config',
  4261. 'layouts/rate-limit-reconfigure2.yaml')
  4262. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  4263. self.waitUntilSettled()
  4264. # D's remaining job should still be queued
  4265. self.assertEqual(len(self.builds), 3)
  4266. self.executor_server.release('project-.*')
  4267. self.waitUntilSettled()
  4268. @simple_layout('layouts/reconfigure-window.yaml')
  4269. def test_reconfigure_window_shrink(self):
  4270. # Test the active window shrinking during reconfiguration
  4271. self.executor_server.hold_jobs_in_build = True
  4272. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4273. A.addApproval('Code-Review', 2)
  4274. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4275. self.waitUntilSettled()
  4276. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  4277. B.addApproval('Code-Review', 2)
  4278. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  4279. self.waitUntilSettled()
  4280. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4281. queue = tenant.layout.pipelines['gate'].queues[0]
  4282. self.assertEqual(queue.window, 20)
  4283. self.assertTrue(len(self.builds), 4)
  4284. self.executor_server.release('job1')
  4285. self.waitUntilSettled()
  4286. self.commitConfigUpdate('org/common-config',
  4287. 'layouts/reconfigure-window2.yaml')
  4288. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  4289. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4290. queue = tenant.layout.pipelines['gate'].queues[0]
  4291. # Even though we have configured a smaller window, the value
  4292. # on the existing shared queue should be used.
  4293. self.assertEqual(queue.window, 20)
  4294. self.assertTrue(len(self.builds), 4)
  4295. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  4296. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4297. queue = tenant.layout.pipelines['gate'].queues[0]
  4298. self.assertEqual(queue.window, 20)
  4299. self.assertTrue(len(self.builds), 4)
  4300. self.executor_server.hold_jobs_in_build = False
  4301. self.executor_server.release()
  4302. self.waitUntilSettled()
  4303. self.assertHistory([
  4304. dict(name='job1', result='SUCCESS', changes='1,1'),
  4305. dict(name='job1', result='SUCCESS', changes='1,1 2,1'),
  4306. dict(name='job2', result='SUCCESS', changes='1,1'),
  4307. dict(name='job2', result='SUCCESS', changes='1,1 2,1'),
  4308. ], ordered=False)
  4309. @simple_layout('layouts/reconfigure-window-fixed.yaml')
  4310. def test_reconfigure_window_fixed(self):
  4311. # Test the active window shrinking during reconfiguration
  4312. self.executor_server.hold_jobs_in_build = True
  4313. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4314. A.addApproval('Code-Review', 2)
  4315. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4316. self.waitUntilSettled()
  4317. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  4318. B.addApproval('Code-Review', 2)
  4319. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  4320. self.waitUntilSettled()
  4321. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4322. queue = tenant.layout.pipelines['gate'].queues[0]
  4323. self.assertEqual(queue.window, 2)
  4324. self.assertEqual(len(self.builds), 4)
  4325. self.waitUntilSettled()
  4326. self.commitConfigUpdate('org/common-config',
  4327. 'layouts/reconfigure-window-fixed2.yaml')
  4328. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  4329. self.waitUntilSettled()
  4330. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4331. queue = tenant.layout.pipelines['gate'].queues[0]
  4332. # Because we have configured a static window, it should
  4333. # be allowed to shrink on reconfiguration.
  4334. self.assertEqual(queue.window, 1)
  4335. # B is outside the window, but still marked active until the
  4336. # next pass through the queue processor.
  4337. self.assertEqual(len(self.builds), 4)
  4338. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  4339. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4340. queue = tenant.layout.pipelines['gate'].queues[0]
  4341. self.assertEqual(queue.window, 1)
  4342. self.waitUntilSettled()
  4343. # B's builds should not be canceled
  4344. self.assertEqual(len(self.builds), 4)
  4345. self.executor_server.hold_jobs_in_build = False
  4346. self.executor_server.release()
  4347. self.waitUntilSettled()
  4348. self.assertHistory([
  4349. dict(name='job1', result='SUCCESS', changes='1,1'),
  4350. dict(name='job2', result='SUCCESS', changes='1,1'),
  4351. dict(name='job1', result='SUCCESS', changes='1,1 2,1'),
  4352. dict(name='job2', result='SUCCESS', changes='1,1 2,1'),
  4353. ], ordered=False)
  4354. @simple_layout('layouts/reconfigure-window-fixed.yaml')
  4355. def test_reconfigure_window_fixed_requests(self):
  4356. # Test the active window shrinking during reconfiguration with
  4357. # outstanding node requests
  4358. self.executor_server.hold_jobs_in_build = True
  4359. # Start the jobs for A
  4360. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4361. A.addApproval('Code-Review', 2)
  4362. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4363. self.waitUntilSettled()
  4364. self.log.debug("A complete")
  4365. # Hold node requests for B
  4366. self.fake_nodepool.pause()
  4367. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  4368. B.addApproval('Code-Review', 2)
  4369. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  4370. self.waitUntilSettled()
  4371. self.log.debug("B complete")
  4372. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4373. queue = tenant.layout.pipelines['gate'].queues[0]
  4374. self.assertEqual(queue.window, 2)
  4375. self.assertEqual(len(self.builds), 2)
  4376. self.waitUntilSettled()
  4377. self.commitConfigUpdate('org/common-config',
  4378. 'layouts/reconfigure-window-fixed2.yaml')
  4379. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  4380. self.waitUntilSettled()
  4381. self.log.debug("Reconfiguration complete")
  4382. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4383. queue = tenant.layout.pipelines['gate'].queues[0]
  4384. # Because we have configured a static window, it should
  4385. # be allowed to shrink on reconfiguration.
  4386. self.assertEqual(queue.window, 1)
  4387. self.assertEqual(len(self.builds), 2)
  4388. # After the previous reconfig, the queue processor will have
  4389. # run and marked B inactive; run another reconfiguration so
  4390. # that we're testing what happens when we reconfigure after
  4391. # the active window having shrunk.
  4392. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  4393. # Unpause the node requests now
  4394. self.fake_nodepool.unpause()
  4395. self.waitUntilSettled()
  4396. self.log.debug("Nodepool unpause complete")
  4397. # Allow A to merge and B to enter the active window and complete
  4398. self.executor_server.hold_jobs_in_build = False
  4399. self.executor_server.release()
  4400. self.waitUntilSettled()
  4401. self.log.debug("Executor unpause complete")
  4402. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4403. queue = tenant.layout.pipelines['gate'].queues[0]
  4404. self.assertEqual(queue.window, 1)
  4405. self.waitUntilSettled()
  4406. self.assertHistory([
  4407. dict(name='job1', result='SUCCESS', changes='1,1'),
  4408. dict(name='job2', result='SUCCESS', changes='1,1'),
  4409. dict(name='job1', result='SUCCESS', changes='1,1 2,1'),
  4410. dict(name='job2', result='SUCCESS', changes='1,1 2,1'),
  4411. ], ordered=False)
  4412. @simple_layout('layouts/reconfigure-remove-add.yaml')
  4413. def test_reconfigure_remove_add(self):
  4414. # Test removing, then adding a job while in queue
  4415. self.executor_server.hold_jobs_in_build = True
  4416. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4417. A.addApproval('Code-Review', 2)
  4418. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4419. self.waitUntilSettled()
  4420. self.assertTrue(len(self.builds), 2)
  4421. self.executor_server.release('job2')
  4422. self.assertTrue(len(self.builds), 1)
  4423. # Remove job2
  4424. self.commitConfigUpdate('org/common-config',
  4425. 'layouts/reconfigure-remove-add2.yaml')
  4426. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  4427. self.assertTrue(len(self.builds), 1)
  4428. # Add job2 back
  4429. self.commitConfigUpdate('org/common-config',
  4430. 'layouts/reconfigure-remove-add.yaml')
  4431. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  4432. self.assertTrue(len(self.builds), 2)
  4433. self.executor_server.hold_jobs_in_build = False
  4434. self.executor_server.release()
  4435. # This will run new builds for B
  4436. self.waitUntilSettled()
  4437. self.assertHistory([
  4438. dict(name='job2', result='SUCCESS', changes='1,1'),
  4439. dict(name='job1', result='SUCCESS', changes='1,1'),
  4440. dict(name='job2', result='SUCCESS', changes='1,1'),
  4441. ], ordered=False)
  4442. def test_worker_update_metadata(self):
  4443. "Test if a worker can send back metadata about itself"
  4444. self.executor_server.hold_jobs_in_build = True
  4445. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4446. A.addApproval('Code-Review', 2)
  4447. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4448. self.waitUntilSettled()
  4449. self.assertEqual(len(self.scheds.first.sched.executor.builds), 1)
  4450. self.log.debug('Current builds:')
  4451. self.log.debug(self.scheds.first.sched.executor.builds)
  4452. start = time.time()
  4453. while True:
  4454. if time.time() - start > 10:
  4455. raise Exception("Timeout waiting for gearman server to report "
  4456. + "back to the client")
  4457. build = list(self.scheds.first.sched.executor.builds.values())[0]
  4458. if build.worker.name == self.executor_server.hostname:
  4459. break
  4460. else:
  4461. time.sleep(0)
  4462. self.log.debug(build)
  4463. self.assertEqual(self.executor_server.hostname, build.worker.name)
  4464. self.executor_server.hold_jobs_in_build = False
  4465. self.executor_server.release()
  4466. self.waitUntilSettled()
  4467. @simple_layout('layouts/footer-message.yaml')
  4468. def test_footer_message(self):
  4469. "Test a pipeline's footer message is correctly added to the report."
  4470. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4471. A.addApproval('Code-Review', 2)
  4472. self.executor_server.failJob('project-test1', A)
  4473. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4474. self.waitUntilSettled()
  4475. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  4476. B.addApproval('Code-Review', 2)
  4477. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  4478. self.waitUntilSettled()
  4479. self.assertEqual(2, len(self.smtp_messages))
  4480. failure_msg = """\
  4481. Build failed. For information on how to proceed, see \
  4482. http://wiki.example.org/Test_Failures"""
  4483. footer_msg = """\
  4484. For CI problems and help debugging, contact ci@example.org"""
  4485. self.assertTrue(self.smtp_messages[0]['body'].startswith(failure_msg))
  4486. self.assertTrue(self.smtp_messages[0]['body'].endswith(footer_msg))
  4487. self.assertFalse(self.smtp_messages[1]['body'].startswith(failure_msg))
  4488. self.assertTrue(self.smtp_messages[1]['body'].endswith(footer_msg))
  4489. @simple_layout('layouts/start-message.yaml')
  4490. def test_start_message(self):
  4491. "Test a pipeline's start message is correctly added to the report."
  4492. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4493. A.addApproval('Code-Review', 2)
  4494. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4495. self.waitUntilSettled()
  4496. self.assertEqual(1, len(self.smtp_messages))
  4497. start_msg = "Jobs started in gate for 1,1."
  4498. self.assertTrue(self.smtp_messages[0]['body'].startswith(start_msg))
  4499. @simple_layout('layouts/unmanaged-project.yaml')
  4500. def test_unmanaged_project_start_message(self):
  4501. "Test start reporting is not done for unmanaged projects."
  4502. self.init_repo("org/project", tag='init')
  4503. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4504. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  4505. self.waitUntilSettled()
  4506. self.assertEqual(0, len(A.messages))
  4507. @simple_layout('layouts/merge-failure.yaml')
  4508. def test_merge_failure_reporters(self):
  4509. """Check that the config is set up correctly"""
  4510. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4511. self.assertEqual(
  4512. "Merge Failed.\n\nThis change or one of its cross-repo "
  4513. "dependencies was unable to be automatically merged with the "
  4514. "current state of its repository. Please rebase the change and "
  4515. "upload a new patchset.",
  4516. tenant.layout.pipelines['check'].merge_failure_message)
  4517. self.assertEqual(
  4518. "The merge failed! For more information...",
  4519. tenant.layout.pipelines['gate'].merge_failure_message)
  4520. self.assertEqual(
  4521. len(tenant.layout.pipelines['check'].merge_failure_actions), 1)
  4522. self.assertEqual(
  4523. len(tenant.layout.pipelines['gate'].merge_failure_actions), 2)
  4524. self.assertTrue(isinstance(
  4525. tenant.layout.pipelines['check'].merge_failure_actions[0],
  4526. gerritreporter.GerritReporter))
  4527. self.assertTrue(
  4528. (
  4529. isinstance(tenant.layout.pipelines['gate'].
  4530. merge_failure_actions[0],
  4531. zuul.driver.smtp.smtpreporter.SMTPReporter) and
  4532. isinstance(tenant.layout.pipelines['gate'].
  4533. merge_failure_actions[1],
  4534. gerritreporter.GerritReporter)
  4535. ) or (
  4536. isinstance(tenant.layout.pipelines['gate'].
  4537. merge_failure_actions[0],
  4538. gerritreporter.GerritReporter) and
  4539. isinstance(tenant.layout.pipelines['gate'].
  4540. merge_failure_actions[1],
  4541. zuul.driver.smtp.smtpreporter.SMTPReporter)
  4542. )
  4543. )
  4544. @skip("Disabled for early v3 development")
  4545. def test_merge_failure_reports(self):
  4546. """Check that when a change fails to merge the correct message is sent
  4547. to the correct reporter"""
  4548. self.updateConfigLayout(
  4549. 'tests/fixtures/layout-merge-failure.yaml')
  4550. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  4551. self.registerJobs()
  4552. # Check a test failure isn't reported to SMTP
  4553. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4554. A.addApproval('Code-Review', 2)
  4555. self.executor_server.failJob('project-test1', A)
  4556. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4557. self.waitUntilSettled()
  4558. self.assertEqual(3, len(self.history)) # 3 jobs
  4559. self.assertEqual(0, len(self.smtp_messages))
  4560. # Check a merge failure is reported to SMTP
  4561. # B should be merged, but C will conflict with B
  4562. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  4563. B.addPatchset(['conflict'])
  4564. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  4565. C.addPatchset(['conflict'])
  4566. B.addApproval('Code-Review', 2)
  4567. C.addApproval('Code-Review', 2)
  4568. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  4569. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  4570. self.waitUntilSettled()
  4571. self.assertEqual(6, len(self.history)) # A and B jobs
  4572. self.assertEqual(1, len(self.smtp_messages))
  4573. self.assertEqual('The merge failed! For more information...',
  4574. self.smtp_messages[0]['body'])
  4575. @skip("Disabled for early v3 development")
  4576. def test_default_merge_failure_reports(self):
  4577. """Check that the default merge failure reports are correct."""
  4578. # A should report success, B should report merge failure.
  4579. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4580. A.addPatchset(['conflict'])
  4581. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  4582. B.addPatchset(['conflict'])
  4583. A.addApproval('Code-Review', 2)
  4584. B.addApproval('Code-Review', 2)
  4585. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4586. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  4587. self.waitUntilSettled()
  4588. self.assertEqual(3, len(self.history)) # A jobs
  4589. self.assertEqual(A.reported, 2)
  4590. self.assertEqual(B.reported, 2)
  4591. self.assertEqual(A.data['status'], 'MERGED')
  4592. self.assertEqual(B.data['status'], 'NEW')
  4593. self.assertIn('Build succeeded', A.messages[1])
  4594. self.assertIn('Merge Failed', B.messages[1])
  4595. self.assertIn('automatically merged', B.messages[1])
  4596. self.assertNotIn('logs.example.com', B.messages[1])
  4597. self.assertNotIn('SKIPPED', B.messages[1])
  4598. def test_client_get_running_jobs(self):
  4599. "Test that the RPC client can get a list of running jobs"
  4600. self.executor_server.hold_jobs_in_build = True
  4601. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4602. A.addApproval('Code-Review', 2)
  4603. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4604. self.waitUntilSettled()
  4605. client = zuul.rpcclient.RPCClient('127.0.0.1',
  4606. self.gearman_server.port)
  4607. self.addCleanup(client.shutdown)
  4608. # Wait for gearman server to send the initial workData back to zuul
  4609. start = time.time()
  4610. while True:
  4611. if time.time() - start > 10:
  4612. raise Exception("Timeout waiting for gearman server to report "
  4613. + "back to the client")
  4614. build = list(self.scheds.first.sched.executor.builds.values())[0]
  4615. if build.worker.name == self.executor_server.hostname:
  4616. break
  4617. else:
  4618. time.sleep(0)
  4619. running_items = client.get_running_jobs()
  4620. self.assertEqual(1, len(running_items))
  4621. running_item = running_items[0]
  4622. self.assertEqual([], running_item['failing_reasons'])
  4623. self.assertEqual([], running_item['items_behind'])
  4624. self.assertEqual('https://review.example.com/1', running_item['url'])
  4625. self.assertIsNone(running_item['item_ahead'])
  4626. self.assertEqual('org/project', running_item['project'])
  4627. self.assertIsNone(running_item['remaining_time'])
  4628. self.assertEqual(True, running_item['active'])
  4629. self.assertEqual('1,1', running_item['id'])
  4630. self.assertEqual(3, len(running_item['jobs']))
  4631. for job in running_item['jobs']:
  4632. if job['name'] == 'project-merge':
  4633. self.assertEqual('project-merge', job['name'])
  4634. self.assertEqual('gate', job['pipeline'])
  4635. self.assertEqual(False, job['retry'])
  4636. self.assertEqual(
  4637. 'stream/{uuid}?logfile=console.log'
  4638. .format(uuid=job['uuid']), job['url'])
  4639. self.assertEqual(
  4640. 'finger://{hostname}/{uuid}'.format(
  4641. hostname=self.executor_server.hostname,
  4642. uuid=job['uuid']),
  4643. job['finger_url'])
  4644. self.assertEqual(2, len(job['worker']))
  4645. self.assertEqual(False, job['canceled'])
  4646. self.assertEqual(True, job['voting'])
  4647. self.assertIsNone(job['result'])
  4648. self.assertEqual('gate', job['pipeline'])
  4649. break
  4650. self.executor_server.hold_jobs_in_build = False
  4651. self.executor_server.release()
  4652. self.waitUntilSettled()
  4653. running_items = client.get_running_jobs()
  4654. self.assertEqual(0, len(running_items))
  4655. @simple_layout('layouts/nonvoting-pipeline.yaml')
  4656. def test_nonvoting_pipeline(self):
  4657. "Test that a nonvoting pipeline (experimental) can still report"
  4658. A = self.fake_gerrit.addFakeChange('org/experimental-project',
  4659. 'master', 'A')
  4660. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  4661. self.waitUntilSettled()
  4662. self.assertEqual(self.getJobFromHistory('project-merge').result,
  4663. 'SUCCESS')
  4664. self.assertEqual(
  4665. self.getJobFromHistory('experimental-project-test').result,
  4666. 'SUCCESS')
  4667. self.assertEqual(A.reported, 1)
  4668. @simple_layout('layouts/disable_at.yaml')
  4669. def test_disable_at(self):
  4670. "Test a pipeline will only report to the disabled trigger when failing"
  4671. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4672. self.assertEqual(3, tenant.layout.pipelines['check'].disable_at)
  4673. self.assertEqual(
  4674. 0, tenant.layout.pipelines['check']._consecutive_failures)
  4675. self.assertFalse(tenant.layout.pipelines['check']._disabled)
  4676. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4677. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  4678. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  4679. D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
  4680. E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
  4681. F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')
  4682. G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')
  4683. H = self.fake_gerrit.addFakeChange('org/project', 'master', 'H')
  4684. I = self.fake_gerrit.addFakeChange('org/project', 'master', 'I')
  4685. J = self.fake_gerrit.addFakeChange('org/project', 'master', 'J')
  4686. K = self.fake_gerrit.addFakeChange('org/project', 'master', 'K')
  4687. self.executor_server.failJob('project-test1', A)
  4688. self.executor_server.failJob('project-test1', B)
  4689. # Let C pass, resetting the counter
  4690. self.executor_server.failJob('project-test1', D)
  4691. self.executor_server.failJob('project-test1', E)
  4692. self.executor_server.failJob('project-test1', F)
  4693. self.executor_server.failJob('project-test1', G)
  4694. self.executor_server.failJob('project-test1', H)
  4695. # I also passes but should only report to the disabled reporters
  4696. self.executor_server.failJob('project-test1', J)
  4697. self.executor_server.failJob('project-test1', K)
  4698. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  4699. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  4700. self.waitUntilSettled()
  4701. self.assertEqual(
  4702. 2, tenant.layout.pipelines['check']._consecutive_failures)
  4703. self.assertFalse(tenant.layout.pipelines['check']._disabled)
  4704. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  4705. self.waitUntilSettled()
  4706. self.assertEqual(
  4707. 0, tenant.layout.pipelines['check']._consecutive_failures)
  4708. self.assertFalse(tenant.layout.pipelines['check']._disabled)
  4709. self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
  4710. self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))
  4711. self.fake_gerrit.addEvent(F.getPatchsetCreatedEvent(1))
  4712. self.waitUntilSettled()
  4713. # We should be disabled now
  4714. self.assertEqual(
  4715. 3, tenant.layout.pipelines['check']._consecutive_failures)
  4716. self.assertTrue(tenant.layout.pipelines['check']._disabled)
  4717. # We need to wait between each of these patches to make sure the
  4718. # smtp messages come back in an expected order
  4719. self.fake_gerrit.addEvent(G.getPatchsetCreatedEvent(1))
  4720. self.waitUntilSettled()
  4721. self.fake_gerrit.addEvent(H.getPatchsetCreatedEvent(1))
  4722. self.waitUntilSettled()
  4723. self.fake_gerrit.addEvent(I.getPatchsetCreatedEvent(1))
  4724. self.waitUntilSettled()
  4725. # The first 6 (ABCDEF) jobs should have reported back to gerrt thus
  4726. # leaving a message on each change
  4727. self.assertEqual(1, len(A.messages))
  4728. self.assertIn('Build failed.', A.messages[0])
  4729. self.assertEqual(1, len(B.messages))
  4730. self.assertIn('Build failed.', B.messages[0])
  4731. self.assertEqual(1, len(C.messages))
  4732. self.assertIn('Build succeeded.', C.messages[0])
  4733. self.assertEqual(1, len(D.messages))
  4734. self.assertIn('Build failed.', D.messages[0])
  4735. self.assertEqual(1, len(E.messages))
  4736. self.assertIn('Build failed.', E.messages[0])
  4737. self.assertEqual(1, len(F.messages))
  4738. self.assertIn('Build failed.', F.messages[0])
  4739. # The last 3 (GHI) would have only reported via smtp.
  4740. self.assertEqual(3, len(self.smtp_messages))
  4741. self.assertEqual(0, len(G.messages))
  4742. self.assertIn('Build failed.', self.smtp_messages[0]['body'])
  4743. self.assertIn(
  4744. 'project-test1 finger://', self.smtp_messages[0]['body'])
  4745. self.assertEqual(0, len(H.messages))
  4746. self.assertIn('Build failed.', self.smtp_messages[1]['body'])
  4747. self.assertIn(
  4748. 'project-test1 finger://', self.smtp_messages[1]['body'])
  4749. self.assertEqual(0, len(I.messages))
  4750. self.assertIn('Build succeeded.', self.smtp_messages[2]['body'])
  4751. self.assertIn(
  4752. 'project-test1 finger://', self.smtp_messages[2]['body'])
  4753. # Now reload the configuration (simulate a HUP) to check the pipeline
  4754. # comes out of disabled
  4755. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  4756. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4757. self.assertEqual(3, tenant.layout.pipelines['check'].disable_at)
  4758. self.assertEqual(
  4759. 0, tenant.layout.pipelines['check']._consecutive_failures)
  4760. self.assertFalse(tenant.layout.pipelines['check']._disabled)
  4761. self.fake_gerrit.addEvent(J.getPatchsetCreatedEvent(1))
  4762. self.fake_gerrit.addEvent(K.getPatchsetCreatedEvent(1))
  4763. self.waitUntilSettled()
  4764. self.assertEqual(
  4765. 2, tenant.layout.pipelines['check']._consecutive_failures)
  4766. self.assertFalse(tenant.layout.pipelines['check']._disabled)
  4767. # J and K went back to gerrit
  4768. self.assertEqual(1, len(J.messages))
  4769. self.assertIn('Build failed.', J.messages[0])
  4770. self.assertEqual(1, len(K.messages))
  4771. self.assertIn('Build failed.', K.messages[0])
  4772. # No more messages reported via smtp
  4773. self.assertEqual(3, len(self.smtp_messages))
  4774. @simple_layout('layouts/one-job-project.yaml')
  4775. def test_one_job_project(self):
  4776. "Test that queueing works with one job"
  4777. A = self.fake_gerrit.addFakeChange('org/one-job-project',
  4778. 'master', 'A')
  4779. B = self.fake_gerrit.addFakeChange('org/one-job-project',
  4780. 'master', 'B')
  4781. A.addApproval('Code-Review', 2)
  4782. B.addApproval('Code-Review', 2)
  4783. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4784. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  4785. self.waitUntilSettled()
  4786. self.assertEqual(A.data['status'], 'MERGED')
  4787. self.assertEqual(A.reported, 2)
  4788. self.assertEqual(B.data['status'], 'MERGED')
  4789. self.assertEqual(B.reported, 2)
  4790. def test_job_aborted(self):
  4791. "Test that if a execute server aborts a job, it is run again"
  4792. self.executor_server.hold_jobs_in_build = True
  4793. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4794. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  4795. self.waitUntilSettled()
  4796. self.executor_server.release('.*-merge')
  4797. self.waitUntilSettled()
  4798. self.assertEqual(len(self.builds), 2)
  4799. # first abort
  4800. self.builds[0].aborted = True
  4801. self.executor_server.release('.*-test*')
  4802. self.waitUntilSettled()
  4803. self.assertEqual(len(self.builds), 1)
  4804. # second abort
  4805. self.builds[0].aborted = True
  4806. self.executor_server.release('.*-test*')
  4807. self.waitUntilSettled()
  4808. self.assertEqual(len(self.builds), 1)
  4809. # third abort
  4810. self.builds[0].aborted = True
  4811. self.executor_server.release('.*-test*')
  4812. self.waitUntilSettled()
  4813. self.assertEqual(len(self.builds), 1)
  4814. # fourth abort
  4815. self.builds[0].aborted = True
  4816. self.executor_server.release('.*-test*')
  4817. self.waitUntilSettled()
  4818. self.assertEqual(len(self.builds), 1)
  4819. self.executor_server.hold_jobs_in_build = False
  4820. self.executor_server.release()
  4821. self.waitUntilSettled()
  4822. self.assertEqual(len(self.history), 7)
  4823. self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)
  4824. self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)
  4825. def test_rerun_on_abort(self):
  4826. "Test that if a execute server fails to run a job, it is run again"
  4827. self.executor_server.hold_jobs_in_build = True
  4828. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4829. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  4830. self.waitUntilSettled()
  4831. self.executor_server.release('.*-merge')
  4832. self.waitUntilSettled()
  4833. self.assertEqual(len(self.builds), 2)
  4834. self.builds[0].requeue = True
  4835. self.executor_server.release('.*-test*')
  4836. self.waitUntilSettled()
  4837. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  4838. items = tenant.layout.pipelines['check'].getAllItems()
  4839. build_set = items[0].current_build_set
  4840. for x in range(3):
  4841. # We should have x+1 retried builds for project-test1
  4842. retry_builds = build_set.getRetryBuildsForJob('project-test1')
  4843. self.assertEqual(len(retry_builds), x + 1)
  4844. for build in retry_builds:
  4845. self.assertEqual(build.retry, True)
  4846. self.assertEqual(build.result, 'RETRY')
  4847. self.assertEqual(len(self.builds), 1,
  4848. 'len of builds at x=%d is wrong' % x)
  4849. self.builds[0].requeue = True
  4850. self.executor_server.release('.*-test1')
  4851. self.waitUntilSettled()
  4852. self.executor_server.hold_jobs_in_build = False
  4853. self.executor_server.release()
  4854. self.waitUntilSettled()
  4855. self.assertEqual(len(self.history), 6)
  4856. self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 2)
  4857. self.assertEqual(A.reported, 1)
  4858. self.assertIn('RETRY_LIMIT', A.messages[0])
  4859. def test_zookeeper_disconnect(self):
  4860. "Test that jobs are executed after a zookeeper disconnect"
  4861. self.fake_nodepool.pause()
  4862. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4863. A.addApproval('Code-Review', 2)
  4864. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4865. self.waitUntilSettled()
  4866. self.scheds.execute(lambda app: app.sched.zk.client.stop())
  4867. self.scheds.execute(lambda app: app.sched.zk.client.start())
  4868. self.fake_nodepool.unpause()
  4869. self.waitUntilSettled()
  4870. self.assertEqual(A.data['status'], 'MERGED')
  4871. self.assertEqual(A.reported, 2)
  4872. def test_zookeeper_disconnect2(self):
  4873. "Test that jobs are executed after a zookeeper disconnect"
  4874. # This tests receiving a ZK disconnect between the arrival of
  4875. # a fulfilled request and when we accept its nodes.
  4876. self.fake_nodepool.pause()
  4877. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4878. A.addApproval('Code-Review', 2)
  4879. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4880. self.waitUntilSettled()
  4881. # We're waiting on the nodepool request to complete. Stop the
  4882. # scheduler from processing further events, then fulfill the
  4883. # nodepool request.
  4884. self.scheds.first.sched.run_handler_lock.acquire()
  4885. # Fulfill the nodepool request.
  4886. self.fake_nodepool.unpause()
  4887. requests = list(self.scheds.first.sched.nodepool.requests.values())
  4888. self.assertEqual(1, len(requests))
  4889. request = requests[0]
  4890. for x in iterate_timeout(30, 'fulfill request'):
  4891. if request.fulfilled:
  4892. break
  4893. id1 = request.id
  4894. # The request is fulfilled, but the scheduler hasn't processed
  4895. # it yet. Reconnect ZK.
  4896. self.scheds.execute(lambda app: app.sched.zk.client.stop())
  4897. self.scheds.execute(lambda app: app.sched.zk.client.start())
  4898. # Allow the scheduler to continue and process the (now
  4899. # out-of-date) notification that nodes are ready.
  4900. self.scheds.first.sched.run_handler_lock.release()
  4901. # It should resubmit the request, once it's fulfilled, we can
  4902. # wait for it to run jobs and settle.
  4903. for x in iterate_timeout(30, 'fulfill request'):
  4904. if request.fulfilled:
  4905. break
  4906. self.waitUntilSettled()
  4907. id2 = request.id
  4908. self.assertEqual(A.data['status'], 'MERGED')
  4909. self.assertEqual(A.reported, 2)
  4910. # Make sure it was resubmitted (the id's should be different).
  4911. self.assertNotEqual(id1, id2)
  4912. def test_nodepool_failure(self):
  4913. "Test that jobs are reported after a nodepool failure"
  4914. self.fake_nodepool.pause()
  4915. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4916. A.addApproval('Code-Review', 2)
  4917. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4918. self.waitUntilSettled()
  4919. req = self.fake_nodepool.getNodeRequests()[0]
  4920. self.fake_nodepool.addFailRequest(req)
  4921. self.fake_nodepool.unpause()
  4922. self.waitUntilSettled()
  4923. self.assertEqual(A.data['status'], 'NEW')
  4924. self.assertEqual(A.reported, 2)
  4925. self.assertIn('project-merge : NODE_FAILURE', A.messages[1])
  4926. self.assertIn('project-test1 : SKIPPED', A.messages[1])
  4927. self.assertIn('project-test2 : SKIPPED', A.messages[1])
  4928. def test_nodepool_resources(self):
  4929. "Test that resources are reported"
  4930. self.executor_server.hold_jobs_in_build = True
  4931. self.fake_nodepool.resources = {
  4932. 'cores': 2,
  4933. 'ram': 1024,
  4934. 'instances': 1,
  4935. }
  4936. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4937. A.addApproval('Code-Review', 2)
  4938. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  4939. self.waitUntilSettled()
  4940. self.executor_server.release('project-merge')
  4941. self.waitUntilSettled()
  4942. # Check that resource usage gauges are reported
  4943. self.assertHistory([
  4944. dict(name='project-merge', result='SUCCESS', changes='1,1'),
  4945. ])
  4946. self.assertReportedStat(
  4947. 'zuul.nodepool.resources.tenant.tenant-one.cores',
  4948. value='2', kind='g')
  4949. self.assertReportedStat(
  4950. 'zuul.nodepool.resources.tenant.tenant-one.ram',
  4951. value='1024', kind='g')
  4952. self.assertReportedStat(
  4953. 'zuul.nodepool.resources.tenant.tenant-one.instances',
  4954. value='1', kind='g')
  4955. self.assertReportedStat(
  4956. 'zuul.nodepool.resources.project.review_example_com/org/project.'
  4957. 'cores', value='2', kind='g')
  4958. self.assertReportedStat(
  4959. 'zuul.nodepool.resources.project.review_example_com/org/project.'
  4960. 'ram', value='1024', kind='g')
  4961. self.assertReportedStat(
  4962. 'zuul.nodepool.resources.project.review_example_com/org/project.'
  4963. 'instances', value='1', kind='g')
  4964. # Check that resource usage counters are reported
  4965. self.assertReportedStat(
  4966. 'zuul.nodepool.resources.tenant.tenant-one.cores',
  4967. kind='c')
  4968. self.assertReportedStat(
  4969. 'zuul.nodepool.resources.tenant.tenant-one.ram',
  4970. kind='c')
  4971. self.assertReportedStat(
  4972. 'zuul.nodepool.resources.tenant.tenant-one.instances',
  4973. kind='c')
  4974. self.assertReportedStat(
  4975. 'zuul.nodepool.resources.project.review_example_com/org/project.'
  4976. 'cores', kind='c')
  4977. self.assertReportedStat(
  4978. 'zuul.nodepool.resources.project.review_example_com/org/project.'
  4979. 'ram', kind='c')
  4980. self.assertReportedStat(
  4981. 'zuul.nodepool.resources.project.review_example_com/org/project.'
  4982. 'instances', kind='c')
  4983. self.executor_server.hold_jobs_in_build = False
  4984. self.executor_server.release()
  4985. self.waitUntilSettled()
  4986. self.assertEqual(A.data['status'], 'MERGED')
  4987. self.assertEqual(A.reported, 2)
  4988. def test_nodepool_pipeline_priority(self):
  4989. "Test that nodes are requested at the correct pipeline priority"
  4990. self.fake_nodepool.pause()
  4991. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  4992. self.fake_gerrit.addEvent(A.getRefUpdatedEvent())
  4993. self.waitUntilSettled()
  4994. B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
  4995. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  4996. self.waitUntilSettled()
  4997. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  4998. C.addApproval('Code-Review', 2)
  4999. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  5000. self.waitUntilSettled()
  5001. reqs = self.fake_nodepool.getNodeRequests()
  5002. # The requests come back sorted by priority. Since we have
  5003. # three requests for the three changes each with a different
  5004. # priority. Also they get a serial number based on order they
  5005. # were received so the number on the endof the oid should map
  5006. # to order submitted.
  5007. # * gate first - high priority - change C
  5008. self.assertEqual(reqs[0]['_oid'], '100-0000000002')
  5009. self.assertEqual(reqs[0]['node_types'], ['label1'])
  5010. # * check second - normal priority - change B
  5011. self.assertEqual(reqs[1]['_oid'], '200-0000000001')
  5012. self.assertEqual(reqs[1]['node_types'], ['label1'])
  5013. # * post third - low priority - change A
  5014. # additionally, the post job defined uses an ubuntu-xenial node,
  5015. # so we include that check just as an extra verification
  5016. self.assertEqual(reqs[2]['_oid'], '300-0000000000')
  5017. self.assertEqual(reqs[2]['node_types'], ['ubuntu-xenial'])
  5018. self.fake_nodepool.unpause()
  5019. self.waitUntilSettled()
  5020. @simple_layout('layouts/two-projects-integrated.yaml')
  5021. def test_nodepool_relative_priority_check(self):
  5022. "Test that nodes are requested at the relative priority"
  5023. self.fake_nodepool.pause()
  5024. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  5025. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5026. self.waitUntilSettled()
  5027. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  5028. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  5029. self.waitUntilSettled()
  5030. C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')
  5031. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  5032. self.waitUntilSettled()
  5033. D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
  5034. self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
  5035. self.waitUntilSettled()
  5036. reqs = self.fake_nodepool.getNodeRequests()
  5037. # The requests come back sorted by priority.
  5038. # Change A, first change for project, high relative priority.
  5039. self.assertEqual(reqs[0]['_oid'], '200-0000000000')
  5040. self.assertEqual(reqs[0]['relative_priority'], 0)
  5041. # Change C, first change for project1, high relative priority.
  5042. self.assertEqual(reqs[1]['_oid'], '200-0000000002')
  5043. self.assertEqual(reqs[1]['relative_priority'], 0)
  5044. # Change B, second change for project, lower relative priority.
  5045. self.assertEqual(reqs[2]['_oid'], '200-0000000001')
  5046. self.assertEqual(reqs[2]['relative_priority'], 1)
  5047. # Change D, first change for project2 shared with project1,
  5048. # lower relative priority than project1.
  5049. self.assertEqual(reqs[3]['_oid'], '200-0000000003')
  5050. self.assertEqual(reqs[3]['relative_priority'], 1)
  5051. # Fulfill only the first request
  5052. self.fake_nodepool.fulfillRequest(reqs[0])
  5053. for x in iterate_timeout(30, 'fulfill request'):
  5054. if len(self.scheds.first.sched.nodepool.requests) < 4:
  5055. break
  5056. self.waitUntilSettled()
  5057. reqs = self.fake_nodepool.getNodeRequests()
  5058. # Change B, now first change for project, equal priority.
  5059. self.assertEqual(reqs[0]['_oid'], '200-0000000001')
  5060. self.assertEqual(reqs[0]['relative_priority'], 0)
  5061. # Change C, now first change for project1, equal priority.
  5062. self.assertEqual(reqs[1]['_oid'], '200-0000000002')
  5063. self.assertEqual(reqs[1]['relative_priority'], 0)
  5064. # Change D, first change for project2 shared with project1,
  5065. # still lower relative priority than project1.
  5066. self.assertEqual(reqs[2]['_oid'], '200-0000000003')
  5067. self.assertEqual(reqs[2]['relative_priority'], 1)
  5068. self.fake_nodepool.unpause()
  5069. self.waitUntilSettled()
  5070. @simple_layout('layouts/two-projects-integrated.yaml')
  5071. def test_nodepool_relative_priority_gate(self):
  5072. "Test that nodes are requested at the relative priority"
  5073. self.fake_nodepool.pause()
  5074. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  5075. A.addApproval('Code-Review', 2)
  5076. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  5077. self.waitUntilSettled()
  5078. B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
  5079. B.addApproval('Code-Review', 2)
  5080. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  5081. self.waitUntilSettled()
  5082. # project does not share a queue with project1 and project2.
  5083. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
  5084. C.addApproval('Code-Review', 2)
  5085. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  5086. self.waitUntilSettled()
  5087. reqs = self.fake_nodepool.getNodeRequests()
  5088. # The requests come back sorted by priority.
  5089. # Change A, first change for shared queue, high relative
  5090. # priority.
  5091. self.assertEqual(reqs[0]['_oid'], '100-0000000000')
  5092. self.assertEqual(reqs[0]['relative_priority'], 0)
  5093. # Change C, first change for independent project, high
  5094. # relative priority.
  5095. self.assertEqual(reqs[1]['_oid'], '100-0000000002')
  5096. self.assertEqual(reqs[1]['relative_priority'], 0)
  5097. # Change B, second change for shared queue, lower relative
  5098. # priority.
  5099. self.assertEqual(reqs[2]['_oid'], '100-0000000001')
  5100. self.assertEqual(reqs[2]['relative_priority'], 1)
  5101. self.fake_nodepool.unpause()
  5102. self.waitUntilSettled()
  5103. def test_nodepool_job_removal(self):
  5104. "Test that nodes are returned unused after job removal"
  5105. self.fake_nodepool.pause()
  5106. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  5107. A.addApproval('Code-Review', 2)
  5108. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  5109. self.waitUntilSettled()
  5110. self.commitConfigUpdate('common-config', 'layouts/no-jobs.yaml')
  5111. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  5112. self.waitUntilSettled()
  5113. self.fake_nodepool.unpause()
  5114. self.waitUntilSettled()
  5115. self.assertEqual(A.data['status'], 'MERGED')
  5116. self.assertEqual(A.reported, 2)
  5117. self.assertHistory([
  5118. dict(name='gate-noop', result='SUCCESS', changes='1,1'),
  5119. ])
  5120. for node in self.fake_nodepool.getNodes():
  5121. self.assertFalse(node['_lock'])
  5122. self.assertEqual(node['state'], 'ready')
  5123. @simple_layout('layouts/multiple-templates.yaml')
  5124. def test_multiple_project_templates(self):
  5125. # Test that applying multiple project templates to a project
  5126. # doesn't alter them when used for a second project.
  5127. A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
  5128. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5129. self.waitUntilSettled()
  5130. build = self.getJobFromHistory('py27')
  5131. self.assertEqual(build.parameters['zuul']['jobtags'], [])
  5132. def test_pending_merge_in_reconfig(self):
  5133. # Test that if we are waiting for an outstanding merge on
  5134. # reconfiguration that we continue to do so.
  5135. self.gearman_server.hold_merge_jobs_in_queue = True
  5136. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  5137. A.setMerged()
  5138. self.fake_gerrit.addEvent(A.getRefUpdatedEvent())
  5139. self.waitUntilSettled()
  5140. self.assertEqual(len(self.scheds.first.sched.merger.jobs), 1)
  5141. gearJob = next(iter(self.scheds.first.sched.merger.jobs))
  5142. self.assertEqual(gearJob.complete, False)
  5143. # Reconfigure while we still have an outstanding merge job
  5144. self.gearman_server.hold_merge_jobs_in_queue = False
  5145. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  5146. (trusted, project1) = tenant.getProject('org/project1')
  5147. self.scheds.first.sched.reconfigureTenant(
  5148. self.scheds.first.sched.abide.tenants['tenant-one'],
  5149. project1, None)
  5150. self.waitUntilSettled()
  5151. # Verify the merge job is still running and that the item is
  5152. # in the pipeline
  5153. self.assertEqual(gearJob.complete, False)
  5154. self.assertEqual(len(self.scheds.first.sched.merger.jobs), 1)
  5155. pipeline = tenant.layout.pipelines['post']
  5156. self.assertEqual(len(pipeline.getAllItems()), 1)
  5157. self.gearman_server.release()
  5158. self.waitUntilSettled()
  5159. self.assertEqual(gearJob.complete, True)
  5160. self.assertEqual(len(self.scheds.first.sched.merger.jobs), 0)
  5161. @simple_layout('layouts/parent-matchers.yaml')
  5162. def test_parent_matchers(self):
  5163. "Test that if a job's parent does not match, the job does not run"
  5164. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  5165. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5166. self.waitUntilSettled()
  5167. self.assertHistory([])
  5168. files = {'foo.txt': ''}
  5169. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
  5170. files=files)
  5171. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  5172. self.waitUntilSettled()
  5173. files = {'bar.txt': ''}
  5174. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C',
  5175. files=files)
  5176. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  5177. self.waitUntilSettled()
  5178. files = {'foo.txt': '', 'bar.txt': ''}
  5179. D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D',
  5180. files=files)
  5181. self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
  5182. self.waitUntilSettled()
  5183. self.assertHistory([
  5184. dict(name='child-job', result='SUCCESS', changes='3,1'),
  5185. dict(name='child-job', result='SUCCESS', changes='4,1'),
  5186. ], ordered=False)
  5187. @simple_layout('layouts/file-matchers.yaml')
  5188. def test_file_matchers(self):
  5189. "Test several file matchers"
  5190. files = {'parent1.txt': ''}
  5191. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
  5192. files=files)
  5193. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5194. self.waitUntilSettled()
  5195. files = {'parent2.txt': ''}
  5196. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
  5197. files=files)
  5198. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  5199. self.waitUntilSettled()
  5200. files = {'child.txt': ''}
  5201. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C',
  5202. files=files)
  5203. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  5204. self.waitUntilSettled()
  5205. files = {'project.txt': ''}
  5206. D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D',
  5207. files=files)
  5208. self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
  5209. self.waitUntilSettled()
  5210. files = {'tests/foo': ''}
  5211. E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E',
  5212. files=files)
  5213. self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))
  5214. self.waitUntilSettled()
  5215. files = {'tests/docs/foo': ''}
  5216. F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F',
  5217. files=files)
  5218. self.fake_gerrit.addEvent(F.getPatchsetCreatedEvent(1))
  5219. self.waitUntilSettled()
  5220. self.assertHistory([
  5221. dict(name='child-job', result='SUCCESS', changes='2,1'),
  5222. dict(name='child-override-job', result='SUCCESS', changes='3,1'),
  5223. dict(name='project-override-job', result='SUCCESS', changes='4,1'),
  5224. dict(name='irr-job', result='SUCCESS', changes='5,1'),
  5225. dict(name='irr-override-job', result='SUCCESS', changes='5,1'),
  5226. dict(name='irr-job', result='SUCCESS', changes='6,1'),
  5227. ], ordered=False)
  5228. def test_trusted_project_dep_on_non_live_untrusted_project(self):
  5229. # Test we get a layout for trusted projects when they depend on
  5230. # non live untrusted projects. This checks against a bug where
  5231. # trusted project config changes can end up in a infinite loop
  5232. # trying to find the right layout.
  5233. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  5234. files = {'zuul.yaml': ''}
  5235. B = self.fake_gerrit.addFakeChange('common-config', 'master', 'B',
  5236. files=files)
  5237. B.setDependsOn(A, 1)
  5238. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  5239. self.waitUntilSettled()
  5240. self.assertHistory([
  5241. dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
  5242. dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'),
  5243. dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'),
  5244. dict(name='project1-project2-integration',
  5245. result='SUCCESS', changes='1,1 2,1'),
  5246. ], ordered=False)
  5247. class TestJobUpdateBrokenConfig(ZuulTestCase):
  5248. tenant_config_file = 'config/job-update-broken/main.yaml'
  5249. def test_fix_check_without_running(self):
  5250. "Test that we can fix a broken check pipeline (don't run the job)"
  5251. in_repo_conf = textwrap.dedent(
  5252. """
  5253. - job:
  5254. name: existing-files
  5255. files:
  5256. - README.txt
  5257. - project-template:
  5258. name: files-template
  5259. check:
  5260. jobs:
  5261. - existing-files
  5262. - noop
  5263. """)
  5264. # When the config is broken, we don't override any files
  5265. # matchers since we don't have a valid basis. Since this
  5266. # doesn't update README.txt, nothing should run.
  5267. file_dict = {'zuul.d/existing.yaml': in_repo_conf}
  5268. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
  5269. files=file_dict)
  5270. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5271. self.waitUntilSettled()
  5272. self.assertHistory([])
  5273. self.assertEqual(A.reported, 1)
  5274. def test_fix_check_with_running(self):
  5275. "Test that we can fix a broken check pipeline (do run the job)"
  5276. in_repo_conf = textwrap.dedent(
  5277. """
  5278. - job:
  5279. name: existing-files
  5280. files:
  5281. - README.txt
  5282. - project-template:
  5283. name: files-template
  5284. check:
  5285. jobs:
  5286. - existing-files
  5287. """)
  5288. # When the config is broken, we don't override any files
  5289. # matchers since we don't have a valid basis. Since this
  5290. # does update README.txt, the job should run.
  5291. file_dict = {'zuul.d/template.yaml': in_repo_conf,
  5292. 'README.txt': ''}
  5293. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
  5294. files=file_dict)
  5295. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5296. self.waitUntilSettled()
  5297. self.assertHistory([
  5298. dict(name='existing-files', result='SUCCESS', changes='1,1'),
  5299. ])
  5300. self.assertEqual(A.reported, 1)
  5301. class TestJobUpdateFileMatcher(ZuulTestCase):
  5302. tenant_config_file = 'config/job-update/main.yaml'
  5303. def test_matchers(self):
  5304. "Test matchers work as expected with no change"
  5305. file_dict = {'README.txt': ''}
  5306. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
  5307. files=file_dict)
  5308. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5309. self.waitUntilSettled()
  5310. file_dict = {'something_else': ''}
  5311. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
  5312. files=file_dict)
  5313. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  5314. self.waitUntilSettled()
  5315. self.assertHistory([
  5316. dict(name='existing-files', result='SUCCESS', changes='1,1'),
  5317. dict(name='existing-irr', result='SUCCESS', changes='2,1'),
  5318. ])
  5319. def test_job_update(self):
  5320. "Test matchers are overridden with a config update"
  5321. in_repo_conf = textwrap.dedent(
  5322. """
  5323. - job:
  5324. name: existing-files
  5325. tags: foo
  5326. - job:
  5327. name: existing-irr
  5328. tags: foo
  5329. """)
  5330. file_dict = {'zuul.d/new.yaml': in_repo_conf}
  5331. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
  5332. files=file_dict)
  5333. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5334. self.waitUntilSettled()
  5335. self.assertHistory([
  5336. dict(name='existing-files', result='SUCCESS', changes='1,1'),
  5337. dict(name='existing-irr', result='SUCCESS', changes='1,1'),
  5338. ], ordered=False)
  5339. def test_job_update_files(self):
  5340. "Test that changes to file matchers themselves don't run jobs"
  5341. # Normally we want to ignore file matchers and run jobs if the
  5342. # job config changes, but if the only thing about the job
  5343. # config that changes *is* the file matchers, then we don't
  5344. # want to run it.
  5345. in_repo_conf = textwrap.dedent(
  5346. """
  5347. - job:
  5348. name: existing-files
  5349. files: 'doesnotexist'
  5350. - job:
  5351. name: existing-irr
  5352. irrelevant-files:
  5353. - README
  5354. - ^zuul.d/.*$
  5355. - newthing
  5356. """)
  5357. file_dict = {'zuul.d/new.yaml': in_repo_conf}
  5358. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
  5359. files=file_dict)
  5360. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5361. self.waitUntilSettled()
  5362. self.assertHistory([])
  5363. def test_new_job(self):
  5364. "Test matchers are overridden when creating a new job"
  5365. in_repo_conf = textwrap.dedent(
  5366. """
  5367. - job:
  5368. name: new-files
  5369. parent: existing-files
  5370. - project:
  5371. check:
  5372. jobs:
  5373. - new-files
  5374. """)
  5375. file_dict = {'zuul.d/new.yaml': in_repo_conf}
  5376. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
  5377. files=file_dict)
  5378. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5379. self.waitUntilSettled()
  5380. self.assertHistory([
  5381. dict(name='new-files', result='SUCCESS', changes='1,1'),
  5382. ])
  5383. def test_patch_series(self):
  5384. "Test that we diff to the nearest layout in a patch series"
  5385. in_repo_conf = textwrap.dedent(
  5386. """
  5387. - job:
  5388. name: new-files1
  5389. parent: existing-files
  5390. - project:
  5391. check:
  5392. jobs:
  5393. - new-files1
  5394. """)
  5395. file_dict = {'zuul.d/new1.yaml': in_repo_conf}
  5396. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
  5397. files=file_dict)
  5398. in_repo_conf = textwrap.dedent(
  5399. """
  5400. - job:
  5401. name: new-files2
  5402. parent: existing-files
  5403. - project:
  5404. check:
  5405. jobs:
  5406. - new-files2
  5407. """)
  5408. file_dict = {'zuul.d/new2.yaml': in_repo_conf}
  5409. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
  5410. files=file_dict)
  5411. B.setDependsOn(A, 1)
  5412. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  5413. self.waitUntilSettled()
  5414. self.assertHistory([
  5415. dict(name='new-files2', result='SUCCESS', changes='1,1 2,1'),
  5416. ])
  5417. def test_disable_match(self):
  5418. "Test matchers are not overridden if we say so"
  5419. in_repo_conf = textwrap.dedent(
  5420. """
  5421. - job:
  5422. name: new-files
  5423. parent: existing-files
  5424. match-on-config-updates: false
  5425. - project:
  5426. check:
  5427. jobs:
  5428. - new-files
  5429. """)
  5430. file_dict = {'zuul.d/new.yaml': in_repo_conf}
  5431. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
  5432. files=file_dict)
  5433. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5434. self.waitUntilSettled()
  5435. self.assertHistory([])
  5436. class TestAmbiguousProjectNames(ZuulTestCase):
  5437. config_file = 'zuul-connections-multiple-gerrits.conf'
  5438. tenant_config_file = 'config/ambiguous-names/main.yaml'
  5439. def test_client_enqueue_canonical(self):
  5440. "Test that the RPC client can enqueue a change using canonical name"
  5441. A = self.fake_review_gerrit.addFakeChange('org/project', 'master', 'A')
  5442. A.addApproval('Code-Review', 2)
  5443. A.addApproval('Approved', 1)
  5444. client = zuul.rpcclient.RPCClient('127.0.0.1',
  5445. self.gearman_server.port)
  5446. self.addCleanup(client.shutdown)
  5447. r = client.enqueue(tenant='tenant-one',
  5448. pipeline='check',
  5449. project='review.example.com/org/project',
  5450. trigger=None,
  5451. change='1,1')
  5452. self.waitUntilSettled()
  5453. self.assertEqual(self.getJobFromHistory('project-merge').result,
  5454. 'SUCCESS')
  5455. self.assertEqual(r, True)
  5456. class TestExecutor(ZuulTestCase):
  5457. tenant_config_file = 'config/single-tenant/main.yaml'
  5458. def assertFinalState(self):
  5459. # In this test, we expect to shut down in a non-final state,
  5460. # so skip these checks.
  5461. pass
  5462. def assertCleanShutdown(self):
  5463. self.log.debug("Assert clean shutdown")
  5464. # After shutdown, make sure no jobs are running
  5465. self.assertEqual({}, self.executor_server.job_workers)
  5466. # Make sure that git.Repo objects have been garbage collected.
  5467. gc.disable()
  5468. try:
  5469. gc.collect()
  5470. for obj in gc.get_objects():
  5471. if isinstance(obj, git.Repo):
  5472. self.log.debug("Leaked git repo object: %s" % repr(obj))
  5473. gc.enable()
  5474. finally:
  5475. gc.enable()
  5476. def test_executor_shutdown(self):
  5477. "Test that the executor can shut down with jobs running"
  5478. self.executor_server.hold_jobs_in_build = True
  5479. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  5480. A.addApproval('Code-Review', 2)
  5481. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  5482. self.waitUntilSettled()
  5483. class TestDependencyGraph(ZuulTestCase):
  5484. tenant_config_file = 'config/dependency-graph/main.yaml'
  5485. def test_dependeny_graph_dispatch_jobs_once(self):
  5486. "Test a job in a dependency graph is queued only once"
  5487. # Job dependencies, starting with A
  5488. # A
  5489. # / \
  5490. # B C
  5491. # / \ / \
  5492. # D F E
  5493. # |
  5494. # G
  5495. self.executor_server.hold_jobs_in_build = True
  5496. change = self.fake_gerrit.addFakeChange(
  5497. 'org/project', 'master', 'change')
  5498. change.addApproval('Code-Review', 2)
  5499. self.fake_gerrit.addEvent(change.addApproval('Approved', 1))
  5500. self.waitUntilSettled()
  5501. self.assertEqual([b.name for b in self.builds], ['A'])
  5502. self.executor_server.release('A')
  5503. self.waitUntilSettled()
  5504. self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
  5505. self.executor_server.release('B')
  5506. self.waitUntilSettled()
  5507. self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
  5508. self.executor_server.release('D')
  5509. self.waitUntilSettled()
  5510. self.assertEqual([b.name for b in self.builds], ['C'])
  5511. self.executor_server.release('C')
  5512. self.waitUntilSettled()
  5513. self.assertEqual(sorted(b.name for b in self.builds), ['E', 'F'])
  5514. self.executor_server.release('F')
  5515. self.waitUntilSettled()
  5516. self.assertEqual(sorted(b.name for b in self.builds), ['E', 'G'])
  5517. self.executor_server.release('G')
  5518. self.waitUntilSettled()
  5519. self.assertEqual([b.name for b in self.builds], ['E'])
  5520. self.executor_server.release('E')
  5521. self.waitUntilSettled()
  5522. self.assertEqual(len(self.builds), 0)
  5523. self.executor_server.hold_jobs_in_build = False
  5524. self.executor_server.release()
  5525. self.waitUntilSettled()
  5526. self.assertEqual(len(self.builds), 0)
  5527. self.assertEqual(len(self.history), 7)
  5528. self.assertEqual(change.data['status'], 'MERGED')
  5529. self.assertEqual(change.reported, 2)
  5530. def test_jobs_launched_only_if_all_dependencies_are_successful(self):
  5531. "Test that a job waits till all dependencies are successful"
  5532. # Job dependencies, starting with A
  5533. # A
  5534. # / \
  5535. # B C*
  5536. # / \ / \
  5537. # D F E
  5538. # |
  5539. # G
  5540. self.executor_server.hold_jobs_in_build = True
  5541. change = self.fake_gerrit.addFakeChange(
  5542. 'org/project', 'master', 'change')
  5543. change.addApproval('Code-Review', 2)
  5544. self.executor_server.failJob('C', change)
  5545. self.fake_gerrit.addEvent(change.addApproval('Approved', 1))
  5546. self.waitUntilSettled()
  5547. self.assertEqual([b.name for b in self.builds], ['A'])
  5548. self.executor_server.release('A')
  5549. self.waitUntilSettled()
  5550. self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
  5551. self.executor_server.release('B')
  5552. self.waitUntilSettled()
  5553. self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
  5554. self.executor_server.release('D')
  5555. self.waitUntilSettled()
  5556. self.assertEqual([b.name for b in self.builds], ['C'])
  5557. self.executor_server.release('C')
  5558. self.waitUntilSettled()
  5559. self.assertEqual(len(self.builds), 0)
  5560. self.executor_server.hold_jobs_in_build = False
  5561. self.executor_server.release()
  5562. self.waitUntilSettled()
  5563. self.assertEqual(len(self.builds), 0)
  5564. self.assertEqual(len(self.history), 4)
  5565. self.assertEqual(change.data['status'], 'NEW')
  5566. self.assertEqual(change.reported, 2)
  5567. @simple_layout('layouts/soft-dependencies-error.yaml')
  5568. def test_soft_dependencies_error(self):
  5569. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  5570. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5571. self.waitUntilSettled()
  5572. self.assertHistory([])
  5573. self.assertEqual(len(A.messages), 1)
  5574. self.assertTrue('Job project-merge not defined' in A.messages[0])
  5575. print(A.messages)
  5576. @simple_layout('layouts/soft-dependencies.yaml')
  5577. def test_soft_dependencies(self):
  5578. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  5579. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5580. self.waitUntilSettled()
  5581. self.assertHistory([
  5582. dict(name='deploy', result='SUCCESS', changes='1,1'),
  5583. ], ordered=False)
  5584. @simple_layout('layouts/soft-dependencies.yaml')
  5585. def test_soft_dependencies_failure(self):
  5586. file_dict = {'main.c': 'test'}
  5587. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
  5588. files=file_dict)
  5589. self.executor_server.failJob('build', A)
  5590. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5591. self.waitUntilSettled()
  5592. self.assertHistory([
  5593. dict(name='build', result='FAILURE', changes='1,1'),
  5594. ], ordered=False)
  5595. self.assertIn('SKIPPED', A.messages[0])
  5596. class TestDuplicatePipeline(ZuulTestCase):
  5597. tenant_config_file = 'config/duplicate-pipeline/main.yaml'
  5598. def test_duplicate_pipelines(self):
  5599. "Test that a change matching multiple pipelines works"
  5600. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  5601. self.fake_gerrit.addEvent(A.getChangeRestoredEvent())
  5602. self.waitUntilSettled()
  5603. self.assertHistory([
  5604. dict(name='project-test1', result='SUCCESS', changes='1,1',
  5605. pipeline='dup1'),
  5606. dict(name='project-test1', result='SUCCESS', changes='1,1',
  5607. pipeline='dup2'),
  5608. ], ordered=False)
  5609. self.assertEqual(len(A.messages), 2)
  5610. if 'dup1' in A.messages[0]:
  5611. self.assertIn('dup1', A.messages[0])
  5612. self.assertNotIn('dup2', A.messages[0])
  5613. self.assertIn('project-test1', A.messages[0])
  5614. self.assertIn('dup2', A.messages[1])
  5615. self.assertNotIn('dup1', A.messages[1])
  5616. self.assertIn('project-test1', A.messages[1])
  5617. else:
  5618. self.assertIn('dup1', A.messages[1])
  5619. self.assertNotIn('dup2', A.messages[1])
  5620. self.assertIn('project-test1', A.messages[1])
  5621. self.assertIn('dup2', A.messages[0])
  5622. self.assertNotIn('dup1', A.messages[0])
  5623. self.assertIn('project-test1', A.messages[0])
  5624. class TestSchedulerRegexProject(ZuulTestCase):
  5625. tenant_config_file = 'config/regex-project/main.yaml'
  5626. def test_regex_project(self):
  5627. "Test that changes are tested in parallel and merged in series"
  5628. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  5629. B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
  5630. C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
  5631. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5632. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  5633. self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
  5634. self.waitUntilSettled()
  5635. # We expect the following builds:
  5636. # - 1 for org/project
  5637. # - 3 for org/project1
  5638. # - 3 for org/project2
  5639. self.assertEqual(len(self.history), 7)
  5640. self.assertEqual(A.reported, 1)
  5641. self.assertEqual(B.reported, 1)
  5642. self.assertEqual(C.reported, 1)
  5643. self.assertHistory([
  5644. dict(name='project-test', result='SUCCESS', changes='1,1'),
  5645. dict(name='project-test1', result='SUCCESS', changes='2,1'),
  5646. dict(name='project-common-test', result='SUCCESS', changes='2,1'),
  5647. dict(name='project-common-test-canonical', result='SUCCESS',
  5648. changes='2,1'),
  5649. dict(name='project-test2', result='SUCCESS', changes='3,1'),
  5650. dict(name='project-common-test', result='SUCCESS', changes='3,1'),
  5651. dict(name='project-common-test-canonical', result='SUCCESS',
  5652. changes='3,1'),
  5653. ], ordered=False)
  5654. class TestSchedulerTemplatedProject(ZuulTestCase):
  5655. tenant_config_file = 'config/templated-project/main.yaml'
  5656. def test_job_from_templates_executed(self):
  5657. "Test whether a job generated via a template can be executed"
  5658. A = self.fake_gerrit.addFakeChange(
  5659. 'org/templated-project', 'master', 'A')
  5660. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5661. self.waitUntilSettled()
  5662. self.assertEqual(self.getJobFromHistory('project-test1').result,
  5663. 'SUCCESS')
  5664. self.assertEqual(self.getJobFromHistory('project-test2').result,
  5665. 'SUCCESS')
  5666. def test_layered_templates(self):
  5667. "Test whether a job generated via a template can be executed"
  5668. A = self.fake_gerrit.addFakeChange(
  5669. 'org/layered-project', 'master', 'A')
  5670. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5671. self.waitUntilSettled()
  5672. self.assertEqual(self.getJobFromHistory('project-test1').result,
  5673. 'SUCCESS')
  5674. self.assertEqual(self.getJobFromHistory('project-test2').result,
  5675. 'SUCCESS')
  5676. self.assertEqual(self.getJobFromHistory('layered-project-test3'
  5677. ).result, 'SUCCESS')
  5678. self.assertEqual(self.getJobFromHistory('layered-project-test4'
  5679. ).result, 'SUCCESS')
  5680. self.assertEqual(self.getJobFromHistory('layered-project-foo-test5'
  5681. ).result, 'SUCCESS')
  5682. self.assertEqual(self.getJobFromHistory('project-test6').result,
  5683. 'SUCCESS')
  5684. def test_unimplied_branch_matchers(self):
  5685. # This tests that there are no implied branch matchers added
  5686. # to project templates in unbranched projects.
  5687. self.create_branch('org/layered-project', 'stable')
  5688. self.fake_gerrit.addEvent(
  5689. self.fake_gerrit.getFakeBranchCreatedEvent(
  5690. 'org/layered-project', 'stable'))
  5691. self.waitUntilSettled()
  5692. A = self.fake_gerrit.addFakeChange(
  5693. 'org/layered-project', 'stable', 'A')
  5694. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5695. self.waitUntilSettled()
  5696. self.assertEqual(self.getJobFromHistory('project-test1').result,
  5697. 'SUCCESS')
  5698. self.log.info(
  5699. self.getJobFromHistory('project-test1').
  5700. parameters['zuul']['_inheritance_path'])
  5701. def test_implied_branch_matchers(self):
  5702. # This tests that there is an implied branch matcher when a
  5703. # template is used on an in-repo project pipeline definition.
  5704. self.create_branch('untrusted-config', 'stable')
  5705. self.fake_gerrit.addEvent(
  5706. self.fake_gerrit.getFakeBranchCreatedEvent(
  5707. 'untrusted-config', 'stable'))
  5708. self.waitUntilSettled()
  5709. A = self.fake_gerrit.addFakeChange(
  5710. 'untrusted-config', 'stable', 'A')
  5711. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5712. self.waitUntilSettled()
  5713. self.assertEqual(self.getJobFromHistory('project-test1').result,
  5714. 'SUCCESS')
  5715. self.log.info(
  5716. self.getJobFromHistory('project-test1').
  5717. parameters['zuul']['_inheritance_path'])
  5718. # Now create a new branch named stable-foo and change the project
  5719. # pipeline
  5720. self.create_branch('untrusted-config', 'stable-foo')
  5721. self.fake_gerrit.addEvent(
  5722. self.fake_gerrit.getFakeBranchCreatedEvent(
  5723. 'untrusted-config', 'stable-foo'))
  5724. self.waitUntilSettled()
  5725. in_repo_conf = textwrap.dedent(
  5726. """
  5727. - project:
  5728. name: untrusted-config
  5729. templates:
  5730. - test-three-and-four
  5731. """)
  5732. file_dict = {'zuul.d/project.yaml': in_repo_conf}
  5733. B = self.fake_gerrit.addFakeChange('untrusted-config', 'stable-foo',
  5734. 'B', files=file_dict)
  5735. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  5736. self.waitUntilSettled()
  5737. self.assertHistory([
  5738. dict(name='project-test1', result='SUCCESS', changes='1,1'),
  5739. dict(name='project-test2', result='SUCCESS', changes='1,1'),
  5740. dict(name='layered-project-test3', result='SUCCESS',
  5741. changes='2,1'),
  5742. dict(name='layered-project-test4', result='SUCCESS',
  5743. changes='2,1'),
  5744. ], ordered=False)
  5745. class TestSchedulerSuccessURL(ZuulTestCase):
  5746. tenant_config_file = 'config/success-url/main.yaml'
  5747. def test_success_url(self):
  5748. "Ensure bad build params are ignored"
  5749. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  5750. self.init_repo('org/docs')
  5751. A = self.fake_gerrit.addFakeChange('org/docs', 'master', 'A')
  5752. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5753. self.waitUntilSettled()
  5754. # Both builds ran: docs-draft-test + docs-draft-test2
  5755. self.assertEqual(len(self.history), 2)
  5756. # Grab build id
  5757. for build in self.history:
  5758. if build.name == 'docs-draft-test':
  5759. uuid = build.uuid[:7]
  5760. elif build.name == 'docs-draft-test2':
  5761. uuid_test2 = build.uuid
  5762. # Two msgs: 'Starting...' + results
  5763. self.assertEqual(len(self.smtp_messages), 2)
  5764. body = self.smtp_messages[1]['body'].splitlines()
  5765. self.assertEqual('Build succeeded.', body[0])
  5766. self.assertIn(
  5767. '- docs-draft-test http://docs-draft.example.org/1/1/1/check/'
  5768. 'docs-draft-test/%s/publish-docs/' % uuid,
  5769. body[2])
  5770. # NOTE: This default URL is currently hard-coded in executor/server.py
  5771. self.assertIn(
  5772. '- docs-draft-test2 finger://{hostname}/{uuid}'.format(
  5773. hostname=self.executor_server.hostname,
  5774. uuid=uuid_test2),
  5775. body[3])
  5776. class TestSchedulerMerges(ZuulTestCase):
  5777. tenant_config_file = 'config/merges/main.yaml'
  5778. def _test_project_merge_mode(self, mode):
  5779. self.executor_server.keep_jobdir = False
  5780. project = 'org/project-%s' % mode
  5781. self.executor_server.hold_jobs_in_build = True
  5782. A = self.fake_gerrit.addFakeChange(project, 'master', 'A')
  5783. B = self.fake_gerrit.addFakeChange(project, 'master', 'B')
  5784. C = self.fake_gerrit.addFakeChange(project, 'master', 'C')
  5785. A.addApproval('Code-Review', 2)
  5786. B.addApproval('Code-Review', 2)
  5787. C.addApproval('Code-Review', 2)
  5788. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  5789. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  5790. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  5791. self.waitUntilSettled()
  5792. build = self.builds[-1]
  5793. path = os.path.join(build.jobdir.src_root, 'review.example.com',
  5794. project)
  5795. repo = git.Repo(path)
  5796. repo_messages = [c.message.strip() for c in repo.iter_commits()]
  5797. repo_messages.reverse()
  5798. self.executor_server.hold_jobs_in_build = False
  5799. self.executor_server.release()
  5800. self.waitUntilSettled()
  5801. return repo_messages
  5802. def _test_merge(self, mode):
  5803. us_path = 'file://' + os.path.join(
  5804. self.upstream_root, 'org/project-%s' % mode)
  5805. expected_messages = [
  5806. 'initial commit',
  5807. 'add content from fixture',
  5808. # the intermediate commits order is nondeterministic
  5809. "Merge commit 'refs/changes/1/2/1' of %s into HEAD" % us_path,
  5810. "Merge commit 'refs/changes/1/3/1' of %s into HEAD" % us_path,
  5811. ]
  5812. result = self._test_project_merge_mode(mode)
  5813. self.assertEqual(result[:2], expected_messages[:2])
  5814. self.assertEqual(result[-2:], expected_messages[-2:])
  5815. def test_project_merge_mode_merge(self):
  5816. self._test_merge('merge')
  5817. def test_project_merge_mode_merge_resolve(self):
  5818. self._test_merge('merge-resolve')
  5819. def test_project_merge_mode_cherrypick(self):
  5820. expected_messages = [
  5821. 'initial commit',
  5822. 'add content from fixture',
  5823. 'A-1',
  5824. 'B-1',
  5825. 'C-1']
  5826. result = self._test_project_merge_mode('cherry-pick')
  5827. self.assertEqual(result, expected_messages)
  5828. def test_merge_branch(self):
  5829. "Test that the right commits are on alternate branches"
  5830. self.create_branch('org/project-merge-branches', 'mp')
  5831. self.fake_gerrit.addEvent(
  5832. self.fake_gerrit.getFakeBranchCreatedEvent(
  5833. 'org/project-merge-branches', 'mp'))
  5834. self.waitUntilSettled()
  5835. self.executor_server.hold_jobs_in_build = True
  5836. A = self.fake_gerrit.addFakeChange(
  5837. 'org/project-merge-branches', 'mp', 'A')
  5838. B = self.fake_gerrit.addFakeChange(
  5839. 'org/project-merge-branches', 'mp', 'B')
  5840. C = self.fake_gerrit.addFakeChange(
  5841. 'org/project-merge-branches', 'mp', 'C')
  5842. A.addApproval('Code-Review', 2)
  5843. B.addApproval('Code-Review', 2)
  5844. C.addApproval('Code-Review', 2)
  5845. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  5846. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  5847. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  5848. self.waitUntilSettled()
  5849. self.executor_server.release('.*-merge')
  5850. self.waitUntilSettled()
  5851. self.executor_server.release('.*-merge')
  5852. self.waitUntilSettled()
  5853. self.executor_server.release('.*-merge')
  5854. self.waitUntilSettled()
  5855. build = self.builds[-1]
  5856. self.assertEqual(build.parameters['zuul']['branch'], 'mp')
  5857. path = os.path.join(build.jobdir.src_root, 'review.example.com',
  5858. 'org/project-merge-branches')
  5859. repo = git.Repo(path)
  5860. repo_messages = [c.message.strip() for c in repo.iter_commits()]
  5861. repo_messages.reverse()
  5862. correct_messages = [
  5863. 'initial commit',
  5864. 'add content from fixture',
  5865. 'mp commit',
  5866. 'A-1', 'B-1', 'C-1']
  5867. self.assertEqual(repo_messages, correct_messages)
  5868. self.executor_server.hold_jobs_in_build = False
  5869. self.executor_server.release()
  5870. self.waitUntilSettled()
  5871. def test_merge_multi_branch(self):
  5872. "Test that dependent changes on multiple branches are merged"
  5873. self.create_branch('org/project-merge-branches', 'mp')
  5874. self.fake_gerrit.addEvent(
  5875. self.fake_gerrit.getFakeBranchCreatedEvent(
  5876. 'org/project-merge-branches', 'mp'))
  5877. self.waitUntilSettled()
  5878. self.executor_server.hold_jobs_in_build = True
  5879. A = self.fake_gerrit.addFakeChange(
  5880. 'org/project-merge-branches', 'master', 'A')
  5881. B = self.fake_gerrit.addFakeChange(
  5882. 'org/project-merge-branches', 'mp', 'B')
  5883. C = self.fake_gerrit.addFakeChange(
  5884. 'org/project-merge-branches', 'master', 'C')
  5885. A.addApproval('Code-Review', 2)
  5886. B.addApproval('Code-Review', 2)
  5887. C.addApproval('Code-Review', 2)
  5888. self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
  5889. self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
  5890. self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
  5891. self.waitUntilSettled()
  5892. job_A = None
  5893. for job in self.builds:
  5894. if 'project-merge' in job.name:
  5895. job_A = job
  5896. path = os.path.join(job_A.jobdir.src_root, 'review.example.com',
  5897. 'org/project-merge-branches')
  5898. repo = git.Repo(path)
  5899. repo_messages = [c.message.strip()
  5900. for c in repo.iter_commits()]
  5901. repo_messages.reverse()
  5902. correct_messages = [
  5903. 'initial commit', 'add content from fixture', 'A-1']
  5904. self.assertEqual(repo_messages, correct_messages)
  5905. self.executor_server.release('.*-merge')
  5906. self.waitUntilSettled()
  5907. job_B = None
  5908. for job in self.builds:
  5909. if 'project-merge' in job.name:
  5910. job_B = job
  5911. path = os.path.join(job_B.jobdir.src_root, 'review.example.com',
  5912. 'org/project-merge-branches')
  5913. repo = git.Repo(path)
  5914. repo_messages = [c.message.strip() for c in repo.iter_commits()]
  5915. repo_messages.reverse()
  5916. correct_messages = [
  5917. 'initial commit', 'add content from fixture', 'mp commit', 'B-1']
  5918. self.assertEqual(repo_messages, correct_messages)
  5919. self.executor_server.release('.*-merge')
  5920. self.waitUntilSettled()
  5921. job_C = None
  5922. for job in self.builds:
  5923. if 'project-merge' in job.name:
  5924. job_C = job
  5925. path = os.path.join(job_C.jobdir.src_root, 'review.example.com',
  5926. 'org/project-merge-branches')
  5927. repo = git.Repo(path)
  5928. repo_messages = [c.message.strip() for c in repo.iter_commits()]
  5929. repo_messages.reverse()
  5930. correct_messages = [
  5931. 'initial commit', 'add content from fixture',
  5932. 'A-1', 'C-1']
  5933. # Ensure the right commits are in the history for this ref
  5934. self.assertEqual(repo_messages, correct_messages)
  5935. self.executor_server.hold_jobs_in_build = False
  5936. self.executor_server.release()
  5937. self.waitUntilSettled()
  5938. class TestSemaphore(ZuulTestCase):
  5939. tenant_config_file = 'config/semaphore/main.yaml'
  5940. def test_semaphore_one(self):
  5941. "Test semaphores with max=1 (mutex)"
  5942. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  5943. self.executor_server.hold_jobs_in_build = True
  5944. # Pause nodepool so we can check the ordering of getting the nodes
  5945. # and aquiring the semaphore.
  5946. self.fake_nodepool.paused = True
  5947. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  5948. B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
  5949. self.assertFalse('test-semaphore' in
  5950. tenant.semaphore_handler.semaphores)
  5951. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  5952. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  5953. self.waitUntilSettled()
  5954. # By default we first lock the semaphore and then get the nodes
  5955. # so at this point the semaphore needs to be aquired.
  5956. self.assertTrue('test-semaphore' in
  5957. tenant.semaphore_handler.semaphores)
  5958. self.fake_nodepool.paused = False
  5959. self.waitUntilSettled()
  5960. self.assertEqual(len(self.builds), 3)
  5961. self.assertEqual(self.builds[0].name, 'project-test1')
  5962. self.assertEqual(self.builds[1].name, 'semaphore-one-test1')
  5963. self.assertEqual(self.builds[2].name, 'project-test1')
  5964. self.executor_server.release('semaphore-one-test1')
  5965. self.waitUntilSettled()
  5966. self.assertEqual(len(self.builds), 3)
  5967. self.assertEqual(self.builds[0].name, 'project-test1')
  5968. self.assertEqual(self.builds[1].name, 'project-test1')
  5969. self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
  5970. self.assertTrue('test-semaphore' in
  5971. tenant.semaphore_handler.semaphores)
  5972. self.executor_server.release('semaphore-one-test2')
  5973. self.waitUntilSettled()
  5974. self.assertEqual(len(self.builds), 3)
  5975. self.assertEqual(self.builds[0].name, 'project-test1')
  5976. self.assertEqual(self.builds[1].name, 'project-test1')
  5977. self.assertEqual(self.builds[2].name, 'semaphore-one-test1')
  5978. self.assertTrue('test-semaphore' in
  5979. tenant.semaphore_handler.semaphores)
  5980. self.executor_server.release('semaphore-one-test1')
  5981. self.waitUntilSettled()
  5982. self.assertEqual(len(self.builds), 3)
  5983. self.assertEqual(self.builds[0].name, 'project-test1')
  5984. self.assertEqual(self.builds[1].name, 'project-test1')
  5985. self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
  5986. self.assertTrue('test-semaphore' in
  5987. tenant.semaphore_handler.semaphores)
  5988. self.executor_server.release('semaphore-one-test2')
  5989. self.waitUntilSettled()
  5990. self.assertEqual(len(self.builds), 2)
  5991. self.assertEqual(self.builds[0].name, 'project-test1')
  5992. self.assertEqual(self.builds[1].name, 'project-test1')
  5993. self.assertFalse('test-semaphore' in
  5994. tenant.semaphore_handler.semaphores)
  5995. self.executor_server.hold_jobs_in_build = False
  5996. self.executor_server.release()
  5997. self.waitUntilSettled()
  5998. self.assertEqual(len(self.builds), 0)
  5999. self.assertEqual(A.reported, 1)
  6000. self.assertEqual(B.reported, 1)
  6001. self.assertFalse('test-semaphore' in
  6002. tenant.semaphore_handler.semaphores)
  6003. def test_semaphore_two(self):
  6004. "Test semaphores with max>1"
  6005. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6006. self.executor_server.hold_jobs_in_build = True
  6007. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  6008. B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
  6009. self.assertFalse('test-semaphore-two' in
  6010. tenant.semaphore_handler.semaphores)
  6011. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6012. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  6013. self.waitUntilSettled()
  6014. self.assertEqual(len(self.builds), 4)
  6015. self.assertEqual(self.builds[0].name, 'project-test1')
  6016. self.assertEqual(self.builds[1].name, 'semaphore-two-test1')
  6017. self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
  6018. self.assertEqual(self.builds[3].name, 'project-test1')
  6019. self.assertTrue('test-semaphore-two' in
  6020. tenant.semaphore_handler.semaphores)
  6021. self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
  6022. 'test-semaphore-two', [])), 2)
  6023. self.executor_server.release('semaphore-two-test1')
  6024. self.waitUntilSettled()
  6025. self.assertEqual(len(self.builds), 4)
  6026. self.assertEqual(self.builds[0].name, 'project-test1')
  6027. self.assertEqual(self.builds[1].name, 'semaphore-two-test2')
  6028. self.assertEqual(self.builds[2].name, 'project-test1')
  6029. self.assertEqual(self.builds[3].name, 'semaphore-two-test1')
  6030. self.assertTrue('test-semaphore-two' in
  6031. tenant.semaphore_handler.semaphores)
  6032. self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
  6033. 'test-semaphore-two', [])), 2)
  6034. self.executor_server.release('semaphore-two-test2')
  6035. self.waitUntilSettled()
  6036. self.assertEqual(len(self.builds), 4)
  6037. self.assertEqual(self.builds[0].name, 'project-test1')
  6038. self.assertEqual(self.builds[1].name, 'project-test1')
  6039. self.assertEqual(self.builds[2].name, 'semaphore-two-test1')
  6040. self.assertEqual(self.builds[3].name, 'semaphore-two-test2')
  6041. self.assertTrue('test-semaphore-two' in
  6042. tenant.semaphore_handler.semaphores)
  6043. self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
  6044. 'test-semaphore-two', [])), 2)
  6045. self.executor_server.release('semaphore-two-test1')
  6046. self.waitUntilSettled()
  6047. self.assertEqual(len(self.builds), 3)
  6048. self.assertEqual(self.builds[0].name, 'project-test1')
  6049. self.assertEqual(self.builds[1].name, 'project-test1')
  6050. self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
  6051. self.assertTrue('test-semaphore-two' in
  6052. tenant.semaphore_handler.semaphores)
  6053. self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
  6054. 'test-semaphore-two', [])), 1)
  6055. self.executor_server.release('semaphore-two-test2')
  6056. self.waitUntilSettled()
  6057. self.assertEqual(len(self.builds), 2)
  6058. self.assertEqual(self.builds[0].name, 'project-test1')
  6059. self.assertEqual(self.builds[1].name, 'project-test1')
  6060. self.assertFalse('test-semaphore-two' in
  6061. tenant.semaphore_handler.semaphores)
  6062. self.executor_server.hold_jobs_in_build = False
  6063. self.executor_server.release()
  6064. self.waitUntilSettled()
  6065. self.assertEqual(len(self.builds), 0)
  6066. self.assertEqual(A.reported, 1)
  6067. self.assertEqual(B.reported, 1)
  6068. def test_semaphore_node_failure(self):
  6069. "Test semaphore and node failure"
  6070. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6071. # Pause nodepool so we can fail the node request later
  6072. self.fake_nodepool.pause()
  6073. A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
  6074. self.assertFalse('test-semaphore' in
  6075. tenant.semaphore_handler.semaphores)
  6076. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6077. self.waitUntilSettled()
  6078. # By default we first lock the semaphore and then get the nodes
  6079. # so at this point the semaphore needs to be aquired.
  6080. self.assertTrue('test-semaphore' in
  6081. tenant.semaphore_handler.semaphores)
  6082. # Fail the node request and unpause
  6083. req = self.fake_nodepool.getNodeRequests()[0]
  6084. self.fake_nodepool.addFailRequest(req)
  6085. self.fake_nodepool.unpause()
  6086. self.waitUntilSettled()
  6087. # At this point the job that holds the semaphore failed with
  6088. # node_failure and the semaphore must be released.
  6089. self.assertFalse('test-semaphore' in
  6090. tenant.semaphore_handler.semaphores)
  6091. self.assertEquals(1, A.reported)
  6092. self.assertIn('semaphore-one-test3 semaphore-one-test3 : NODE_FAILURE',
  6093. A.messages[0])
  6094. def test_semaphore_resources_first(self):
  6095. "Test semaphores with max=1 (mutex) and get resources first"
  6096. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6097. self.executor_server.hold_jobs_in_build = True
  6098. # Pause nodepool so we can check the ordering of getting the nodes
  6099. # and aquiring the semaphore.
  6100. self.fake_nodepool.paused = True
  6101. A = self.fake_gerrit.addFakeChange('org/project3', 'master', 'A')
  6102. B = self.fake_gerrit.addFakeChange('org/project3', 'master', 'B')
  6103. self.assertFalse('test-semaphore' in
  6104. tenant.semaphore_handler.semaphores)
  6105. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6106. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  6107. self.waitUntilSettled()
  6108. # Here we first get the resources and then lock the semaphore
  6109. # so at this point the semaphore should not be aquired.
  6110. self.assertFalse('test-semaphore' in
  6111. tenant.semaphore_handler.semaphores)
  6112. self.fake_nodepool.paused = False
  6113. self.waitUntilSettled()
  6114. self.assertEqual(len(self.builds), 3)
  6115. self.assertEqual(self.builds[0].name, 'project-test1')
  6116. self.assertEqual(self.builds[1].name,
  6117. 'semaphore-one-test1-resources-first')
  6118. self.assertEqual(self.builds[2].name, 'project-test1')
  6119. self.executor_server.release('semaphore-one-test1')
  6120. self.waitUntilSettled()
  6121. self.assertEqual(len(self.builds), 3)
  6122. self.assertEqual(self.builds[0].name, 'project-test1')
  6123. self.assertEqual(self.builds[1].name, 'project-test1')
  6124. self.assertEqual(self.builds[2].name,
  6125. 'semaphore-one-test2-resources-first')
  6126. self.assertTrue('test-semaphore' in
  6127. tenant.semaphore_handler.semaphores)
  6128. self.executor_server.hold_jobs_in_build = False
  6129. self.executor_server.release()
  6130. self.waitUntilSettled()
  6131. def test_semaphore_resources_first_node_failure(self):
  6132. "Test semaphore and node failure"
  6133. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6134. # Pause nodepool so we can fail the node request later
  6135. self.fake_nodepool.pause()
  6136. A = self.fake_gerrit.addFakeChange('org/project4', 'master', 'A')
  6137. self.assertFalse('test-semaphore' in
  6138. tenant.semaphore_handler.semaphores)
  6139. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6140. self.waitUntilSettled()
  6141. # With resources first we first get the nodes so at this point the
  6142. # semaphore must not be aquired.
  6143. self.assertFalse('test-semaphore' in
  6144. tenant.semaphore_handler.semaphores)
  6145. # Fail the node request and unpause
  6146. req = self.fake_nodepool.getNodeRequests()[0]
  6147. self.fake_nodepool.addFailRequest(req)
  6148. self.fake_nodepool.unpause()
  6149. self.waitUntilSettled()
  6150. # At this point the job should never have acuired a semaphore so check
  6151. # that it still has not locked a semaphore.
  6152. self.assertFalse('test-semaphore' in
  6153. tenant.semaphore_handler.semaphores)
  6154. self.assertEquals(1, A.reported)
  6155. self.assertIn('semaphore-one-test1-resources-first : NODE_FAILURE',
  6156. A.messages[0])
  6157. def test_semaphore_zk_error(self):
  6158. "Test semaphore release with zk error"
  6159. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6160. A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
  6161. self.assertFalse('test-semaphore' in
  6162. tenant.semaphore_handler.semaphores)
  6163. # Simulate a single zk error in useNodeSet
  6164. orig_useNodeSet = self.scheds.first.sched.nodepool.useNodeSet
  6165. def broken_use_nodeset(nodeset, build_set=None, event=None):
  6166. # restore original useNodeSet
  6167. self.scheds.first.sched.nodepool.useNodeSet = orig_useNodeSet
  6168. raise NoNodeError()
  6169. self.scheds.first.sched.nodepool.useNodeSet = broken_use_nodeset
  6170. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6171. self.waitUntilSettled()
  6172. # The semaphore should be released
  6173. self.assertFalse('test-semaphore' in
  6174. tenant.semaphore_handler.semaphores)
  6175. # cleanup the queue
  6176. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6177. self.waitUntilSettled()
  6178. def test_semaphore_abandon(self):
  6179. "Test abandon with job semaphores"
  6180. self.executor_server.hold_jobs_in_build = True
  6181. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6182. check_pipeline = tenant.layout.pipelines['check']
  6183. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  6184. self.assertFalse('test-semaphore' in
  6185. tenant.semaphore_handler.semaphores)
  6186. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6187. self.waitUntilSettled()
  6188. self.assertTrue('test-semaphore' in
  6189. tenant.semaphore_handler.semaphores)
  6190. self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
  6191. self.waitUntilSettled()
  6192. # The check pipeline should be empty
  6193. items = check_pipeline.getAllItems()
  6194. self.assertEqual(len(items), 0)
  6195. # The semaphore should be released
  6196. self.assertFalse('test-semaphore' in
  6197. tenant.semaphore_handler.semaphores)
  6198. self.executor_server.hold_jobs_in_build = False
  6199. self.executor_server.release()
  6200. self.waitUntilSettled()
  6201. def test_semaphore_abandon_pending_node_request(self):
  6202. "Test abandon with job semaphores and pending node request"
  6203. self.executor_server.hold_jobs_in_build = True
  6204. # Pause nodepool so we can check the ordering of getting the nodes
  6205. # and aquiring the semaphore.
  6206. self.fake_nodepool.paused = True
  6207. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6208. check_pipeline = tenant.layout.pipelines['check']
  6209. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  6210. self.assertFalse('test-semaphore' in
  6211. tenant.semaphore_handler.semaphores)
  6212. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6213. self.waitUntilSettled()
  6214. self.assertTrue('test-semaphore' in
  6215. tenant.semaphore_handler.semaphores)
  6216. self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
  6217. self.waitUntilSettled()
  6218. # The check pipeline should be empty
  6219. items = check_pipeline.getAllItems()
  6220. self.assertEqual(len(items), 0)
  6221. # The semaphore should be released
  6222. self.assertFalse('test-semaphore' in
  6223. tenant.semaphore_handler.semaphores)
  6224. self.executor_server.hold_jobs_in_build = False
  6225. self.fake_nodepool.paused = False
  6226. self.executor_server.release()
  6227. self.waitUntilSettled()
  6228. def test_semaphore_abandon_pending_execution(self):
  6229. "Test abandon with job semaphores and pending job execution"
  6230. # Pause the executor so it doesn't take any jobs.
  6231. self.executor_server.pause()
  6232. # Start merger as the paused executor won't take merge jobs.
  6233. self._startMerger()
  6234. # Pause nodepool so we can wait on the node requests and fulfill them
  6235. # in a controlled manner.
  6236. self.fake_nodepool.paused = True
  6237. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6238. check_pipeline = tenant.layout.pipelines['check']
  6239. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  6240. self.assertFalse('test-semaphore' in
  6241. tenant.semaphore_handler.semaphores)
  6242. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6243. self.waitUntilSettled()
  6244. self.assertEqual(len(self.scheds.first.sched.nodepool.requests), 2)
  6245. # Now unpause nodepool to fulfill the node requests. We cannot use
  6246. # waitUntilSettled here because the executor is paused.
  6247. self.fake_nodepool.paused = False
  6248. for _ in iterate_timeout(30, 'fulfill node requests'):
  6249. if len(self.scheds.first.sched.nodepool.requests) == 0:
  6250. break
  6251. self.assertTrue('test-semaphore' in
  6252. tenant.semaphore_handler.semaphores)
  6253. self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
  6254. self.waitUntilSettled()
  6255. # The check pipeline should be empty
  6256. items = check_pipeline.getAllItems()
  6257. self.assertEqual(len(items), 0)
  6258. # The semaphore should be released
  6259. self.assertFalse('test-semaphore' in
  6260. tenant.semaphore_handler.semaphores)
  6261. self.executor_server.release()
  6262. self.waitUntilSettled()
  6263. def test_semaphore_new_patchset(self):
  6264. "Test new patchset with job semaphores"
  6265. self.executor_server.hold_jobs_in_build = True
  6266. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6267. check_pipeline = tenant.layout.pipelines['check']
  6268. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  6269. self.assertFalse('test-semaphore' in
  6270. tenant.semaphore_handler.semaphores)
  6271. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6272. self.waitUntilSettled()
  6273. self.assertTrue('test-semaphore' in
  6274. tenant.semaphore_handler.semaphores)
  6275. semaphore = tenant.semaphore_handler.semaphores['test-semaphore']
  6276. self.assertEqual(len(semaphore), 1)
  6277. A.addPatchset()
  6278. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
  6279. self.waitUntilSettled()
  6280. self.assertTrue('test-semaphore' in
  6281. tenant.semaphore_handler.semaphores)
  6282. semaphore = tenant.semaphore_handler.semaphores['test-semaphore']
  6283. self.assertEqual(len(semaphore), 1)
  6284. items = check_pipeline.getAllItems()
  6285. self.assertEqual(items[0].change.number, '1')
  6286. self.assertEqual(items[0].change.patchset, '2')
  6287. self.assertTrue(items[0].live)
  6288. self.executor_server.hold_jobs_in_build = False
  6289. self.executor_server.release()
  6290. self.waitUntilSettled()
  6291. # The semaphore should be released
  6292. self.assertFalse('test-semaphore' in
  6293. tenant.semaphore_handler.semaphores)
  6294. def test_semaphore_reconfigure(self):
  6295. "Test reconfigure with job semaphores"
  6296. self.executor_server.hold_jobs_in_build = True
  6297. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6298. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  6299. self.assertFalse('test-semaphore' in
  6300. tenant.semaphore_handler.semaphores)
  6301. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6302. self.waitUntilSettled()
  6303. self.assertTrue('test-semaphore' in
  6304. tenant.semaphore_handler.semaphores)
  6305. # reconfigure without layout change
  6306. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  6307. self.waitUntilSettled()
  6308. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6309. # semaphore still must be held
  6310. self.assertTrue('test-semaphore' in
  6311. tenant.semaphore_handler.semaphores)
  6312. self.commitConfigUpdate(
  6313. 'common-config',
  6314. 'config/semaphore/zuul-reconfiguration.yaml')
  6315. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  6316. self.waitUntilSettled()
  6317. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6318. self.executor_server.release('project-test1')
  6319. self.waitUntilSettled()
  6320. # There should be no builds anymore
  6321. self.assertEqual(len(self.builds), 0)
  6322. # The semaphore should be released
  6323. self.assertFalse('test-semaphore' in
  6324. tenant.semaphore_handler.semaphores)
  6325. def test_semaphore_reconfigure_job_removal(self):
  6326. "Test job removal during reconfiguration with semaphores"
  6327. self.executor_server.hold_jobs_in_build = True
  6328. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6329. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  6330. self.assertFalse('test-semaphore' in
  6331. tenant.semaphore_handler.semaphores)
  6332. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6333. self.waitUntilSettled()
  6334. self.assertTrue('test-semaphore' in
  6335. tenant.semaphore_handler.semaphores)
  6336. self.commitConfigUpdate(
  6337. 'common-config',
  6338. 'config/semaphore/git/common-config/zuul-remove-job.yaml')
  6339. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  6340. self.waitUntilSettled()
  6341. # Release job project-test1 which should be the only job left
  6342. self.executor_server.release('project-test1')
  6343. self.waitUntilSettled()
  6344. # The check pipeline should be empty
  6345. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6346. check_pipeline = tenant.layout.pipelines['check']
  6347. items = check_pipeline.getAllItems()
  6348. self.assertEqual(len(items), 0)
  6349. # The semaphore should be released
  6350. self.assertFalse('test-semaphore' in
  6351. tenant.semaphore_handler.semaphores)
  6352. self.executor_server.hold_jobs_in_build = False
  6353. self.executor_server.release()
  6354. self.waitUntilSettled()
  6355. def test_semaphore_reconfigure_job_removal_pending_node_request(self):
  6356. """
  6357. Test job removal during reconfiguration with semaphores and pending
  6358. node request.
  6359. """
  6360. self.executor_server.hold_jobs_in_build = True
  6361. # Pause nodepool so we can block the job in node request state during
  6362. # reconfiguration.
  6363. self.fake_nodepool.pause()
  6364. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6365. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  6366. self.assertFalse('test-semaphore' in
  6367. tenant.semaphore_handler.semaphores)
  6368. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6369. self.waitUntilSettled()
  6370. self.assertTrue('test-semaphore' in
  6371. tenant.semaphore_handler.semaphores)
  6372. self.commitConfigUpdate(
  6373. 'common-config',
  6374. 'config/semaphore/git/common-config/zuul-remove-job.yaml')
  6375. self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
  6376. self.waitUntilSettled()
  6377. # Now we can unpause nodepool
  6378. self.fake_nodepool.unpause()
  6379. self.waitUntilSettled()
  6380. # Release job project-test1 which should be the only job left
  6381. self.executor_server.release('project-test1')
  6382. self.waitUntilSettled()
  6383. # The check pipeline should be empty
  6384. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6385. check_pipeline = tenant.layout.pipelines['check']
  6386. items = check_pipeline.getAllItems()
  6387. self.assertEqual(len(items), 0)
  6388. # The semaphore should be released
  6389. self.assertFalse('test-semaphore' in
  6390. tenant.semaphore_handler.semaphores)
  6391. self.executor_server.hold_jobs_in_build = False
  6392. self.executor_server.release()
  6393. self.waitUntilSettled()
  6394. class TestSemaphoreMultiTenant(ZuulTestCase):
  6395. tenant_config_file = 'config/multi-tenant-semaphore/main.yaml'
  6396. def test_semaphore_tenant_isolation(self):
  6397. "Test semaphores in multiple tenants"
  6398. self.waitUntilSettled()
  6399. tenant_one = self.scheds.first.sched.abide.tenants.get('tenant-one')
  6400. tenant_two = self.scheds.first.sched.abide.tenants.get('tenant-two')
  6401. self.executor_server.hold_jobs_in_build = True
  6402. A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
  6403. B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
  6404. C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
  6405. D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
  6406. E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E')
  6407. self.assertFalse('test-semaphore' in
  6408. tenant_one.semaphore_handler.semaphores)
  6409. self.assertFalse('test-semaphore' in
  6410. tenant_two.semaphore_handler.semaphores)
  6411. # add patches to project1 of tenant-one
  6412. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  6413. self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
  6414. self.waitUntilSettled()
  6415. # one build of project1-test1 must run
  6416. # semaphore of tenant-one must be acquired once
  6417. # semaphore of tenant-two must not be acquired
  6418. self