zuul
/
zuul
Code Issues Proposed changes
The Gatekeeper, or a project gating system
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
6999 Commits
1 Branch
27 MiB
 
 
 
Tree: f05adde83e
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'f05adde83e'
${ noResults }
zuul/tests/unit/test_scheduler.py

8436 lines
344 KiB
Raw Blame History 

  1. # Copyright 2012 Hewlett-Packard Development Company, L.P.

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License"); you may

  4. # not use this file except in compliance with the License. You may obtain

  5. # a copy of the License at

  6. #

  7. #      http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT

  11. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the

  12. # License for the specific language governing permissions and limitations

  13. # under the License.

  14. 

  15. import gc

  16. import json

  17. import textwrap

  18. 

  19. import os

  20. import shutil

  21. import socket

  22. import time

  23. from unittest import mock

  24. from unittest import skip

  25. from kazoo.exceptions import NoNodeError

  26. 

  27. import git

  28. import testtools

  29. from zuul.scheduler import Scheduler

  30. 

  31. import zuul.change_matcher

  32. from zuul.driver.gerrit import gerritreporter

  33. import zuul.scheduler

  34. import zuul.rpcclient

  35. import zuul.model

  36. 

  37. from tests.base import (

  38.     SSLZuulTestCase,

  39.     ZuulTestCase,

  40.     repack_repo,

  41.     simple_layout,

  42.     iterate_timeout,

  43. )

  44. 

  45. 

  46. class TestSchedulerSSL(SSLZuulTestCase):

  47.     tenant_config_file = 'config/single-tenant/main.yaml'

  48. 

  49.     def test_jobs_executed(self):

  50.         "Test that jobs are executed and a change is merged"

  51.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  52.         A.addApproval('Code-Review', 2)

  53.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  54.         self.waitUntilSettled()

  55.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  56.                          'SUCCESS')

  57.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  58.                          'SUCCESS')

  59.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  60.                          'SUCCESS')

  61.         self.assertEqual(A.data['status'], 'MERGED')

  62.         self.assertEqual(A.reported, 2)

  63.         self.assertEqual(self.getJobFromHistory('project-test1').node,

  64.                          'label1')

  65.         self.assertEqual(self.getJobFromHistory('project-test2').node,

  66.                          'label1')

  67. 

  68. 

  69. class TestSchedulerZone(ZuulTestCase):

  70.     tenant_config_file = 'config/single-tenant/main.yaml'

  71. 

  72.     def setUp(self):

  73.         super(TestSchedulerZone, self).setUp()

  74.         self.fake_nodepool.attributes = {'executor-zone': 'test-provider.vpn'}

  75. 

  76.     def setup_config(self, config_file: str):

  77.         config = super(TestSchedulerZone, self).setup_config(config_file)

  78.         config.set('executor', 'zone', 'test-provider.vpn')

  79.         return config

  80. 

  81.     def test_jobs_executed(self):

  82.         "Test that jobs are executed and a change is merged per zone"

  83.         self.gearman_server.hold_jobs_in_queue = True

  84.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  85.         A.addApproval('Code-Review', 2)

  86.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  87.         self.waitUntilSettled()

  88. 

  89.         queue = self.gearman_server.getQueue()

  90.         self.assertEqual(len(self.builds), 0)

  91.         self.assertEqual(len(queue), 1)

  92.         self.assertEqual(b'executor:execute:test-provider.vpn', queue[0].name)

  93. 

  94.         self.gearman_server.hold_jobs_in_queue = False

  95.         self.gearman_server.release()

  96.         self.waitUntilSettled()

  97. 

  98.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  99.                          'SUCCESS')

  100.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  101.                          'SUCCESS')

  102.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  103.                          'SUCCESS')

  104.         self.assertEqual(A.data['status'], 'MERGED')

  105.         self.assertEqual(A.reported, 2)

  106.         self.assertEqual(self.getJobFromHistory('project-test1').node,

  107.                          'label1')

  108.         self.assertEqual(self.getJobFromHistory('project-test2').node,

  109.                          'label1')

  110. 

  111. 

  112. class TestAuthorizeViaRPC(ZuulTestCase):

  113.     tenant_config_file = 'config/authorization/single-tenant/main.yaml'

  114. 

  115.     def test_authorize_via_rpc(self):

  116.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  117.                                           self.gearman_server.port)

  118.         self.addCleanup(client.shutdown)

  119.         claims = {'__zuul_uid_claim': 'venkman'}

  120.         authorized = client.submitJob('zuul:authorize_user',

  121.                                       {'tenant': 'tenant-one',

  122.                                        'claims': claims}).data[0]

  123.         self.assertTrue(json.loads(authorized))

  124.         claims = {'sub': 'gozer'}

  125.         authorized = client.submitJob('zuul:authorize_user',

  126.                                       {'tenant': 'tenant-one',

  127.                                        'claims': claims}).data[0]

  128.         self.assertTrue(not json.loads(authorized))

  129.         claims = {'sub': 'stantz',

  130.                   'iss': 'columbia.edu'}

  131.         authorized = client.submitJob('zuul:authorize_user',

  132.                                       {'tenant': 'tenant-one',

  133.                                        'claims': claims}).data[0]

  134.         self.assertTrue(json.loads(authorized))

  135.         claims = {'sub': 'slimer',

  136.                   'groups': ['ghostbusters', 'ectoplasms']}

  137.         authorized = client.submitJob('zuul:authorize_user',

  138.                                       {'tenant': 'tenant-one',

  139.                                        'claims': claims}).data[0]

  140.         self.assertTrue(json.loads(authorized))

  141. 

  142. 

  143. class TestAuthorizeWithTemplatingViaRPC(ZuulTestCase):

  144.     tenant_config_file = 'config/authorization/rules-templating/main.yaml'

  145. 

  146.     def test_authorize_via_rpc(self):

  147.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  148.                                           self.gearman_server.port)

  149.         self.addCleanup(client.shutdown)

  150.         tenants = ['tenant-zero', 'tenant-one', 'tenant-two']

  151.         for t_claim in tenants:

  152.             claims = {'groups': [t_claim, ]}

  153.             for tenant in tenants:

  154.                 authorized = client.submitJob('zuul:authorize_user',

  155.                                               {'tenant': tenant,

  156.                                                'claims': claims}).data[0]

  157.                 if t_claim == tenant:

  158.                     self.assertTrue(

  159.                         json.loads(authorized),

  160.                         "Failed for t_claim: %s, tenant: %s" % (t_claim,

  161.                                                                 tenant))

  162.                 else:

  163.                     self.assertTrue(

  164.                         not json.loads(authorized),

  165.                         "Failed for t_claim: %s, tenant: %s" % (t_claim,

  166.                                                                 tenant))

  167. 

  168. 

  169. class TestSchedulerAutoholdHoldExpiration(ZuulTestCase):

  170.     '''

  171.     This class of tests validates the autohold node expiration values

  172.     are set correctly via zuul config or from a custom value.

  173.     '''

  174.     config_file = 'zuul-hold-expiration.conf'

  175.     tenant_config_file = 'config/single-tenant/main.yaml'

  176. 

  177.     @simple_layout('layouts/autohold.yaml')

  178.     def test_autohold_max_hold_default(self):

  179.         '''

  180.         Test that the hold request node expiration will default to the

  181.         value specified in the configuration file.

  182.         '''

  183.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  184.                                           self.gearman_server.port)

  185.         self.addCleanup(client.shutdown)

  186. 

  187.         # Add a autohold with no hold expiration.

  188.         r = client.autohold('tenant-one', 'org/project', 'project-test2',

  189.                             "", "", "reason text", 1)

  190.         self.assertTrue(r)

  191. 

  192.         # There should be a record in ZooKeeper

  193.         request_list = self.scheds.first.sched.zk.getHoldRequests()

  194.         self.assertEqual(1, len(request_list))

  195.         request = self.scheds.first.sched.zk.getHoldRequest(request_list[0])

  196.         self.assertIsNotNone(request)

  197.         self.assertEqual('tenant-one', request.tenant)

  198.         self.assertEqual('review.example.com/org/project', request.project)

  199.         self.assertEqual('project-test2', request.job)

  200.         self.assertEqual('reason text', request.reason)

  201.         self.assertEqual(1, request.max_count)

  202.         self.assertEqual(0, request.current_count)

  203.         self.assertEqual([], request.nodes)

  204.         # This should be the default value from the zuul config file.

  205.         self.assertEqual(1800, request.node_expiration)

  206. 

  207.     @simple_layout('layouts/autohold.yaml')

  208.     def test_autohold_max_hold_custom(self):

  209.         '''

  210.         Test that the hold request node expiration will be set to the custom

  211.         value specified in the request.

  212.         '''

  213.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  214.                                           self.gearman_server.port)

  215.         self.addCleanup(client.shutdown)

  216. 

  217.         # Add a autohold with a custom hold expiration.

  218.         r = client.autohold('tenant-one', 'org/project', 'project-test2',

  219.                             "", "", "reason text", 1, 500)

  220.         self.assertTrue(r)

  221. 

  222.         # There should be a record in ZooKeeper

  223.         request_list = self.scheds.first.sched.zk.getHoldRequests()

  224.         self.assertEqual(1, len(request_list))

  225.         request = self.scheds.first.sched.zk.getHoldRequest(request_list[0])

  226.         self.assertIsNotNone(request)

  227.         self.assertEqual('tenant-one', request.tenant)

  228.         self.assertEqual('review.example.com/org/project', request.project)

  229.         self.assertEqual('project-test2', request.job)

  230.         self.assertEqual('reason text', request.reason)

  231.         self.assertEqual(1, request.max_count)

  232.         self.assertEqual(0, request.current_count)

  233.         self.assertEqual([], request.nodes)

  234.         # This should be the value from the user request.

  235.         self.assertEqual(500, request.node_expiration)

  236. 

  237.     @simple_layout('layouts/autohold.yaml')

  238.     def test_autohold_max_hold_custom_invalid(self):

  239.         '''

  240.         Test that if the custom hold request node expiration is higher than our

  241.         configured max, it will be lowered to the max.

  242.         '''

  243.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  244.                                           self.gearman_server.port)

  245.         self.addCleanup(client.shutdown)

  246. 

  247.         # Add a autohold with a custom hold expiration that is higher than our

  248.         # configured max.

  249.         r = client.autohold('tenant-one', 'org/project', 'project-test2',

  250.                             "", "", "reason text", 1, 10000)

  251.         self.assertTrue(r)

  252. 

  253.         # There should be a record in ZooKeeper

  254.         request_list = self.scheds.first.sched.zk.getHoldRequests()

  255.         self.assertEqual(1, len(request_list))

  256.         request = self.scheds.first.sched.zk.getHoldRequest(request_list[0])

  257.         self.assertIsNotNone(request)

  258.         self.assertEqual('tenant-one', request.tenant)

  259.         self.assertEqual('review.example.com/org/project', request.project)

  260.         self.assertEqual('project-test2', request.job)

  261.         self.assertEqual('reason text', request.reason)

  262.         self.assertEqual(1, request.max_count)

  263.         self.assertEqual(0, request.current_count)

  264.         self.assertEqual([], request.nodes)

  265.         # This should be the max value from the zuul config file.

  266.         self.assertEqual(3600, request.node_expiration)

  267. 

  268. 

  269. class TestScheduler(ZuulTestCase):

  270.     tenant_config_file = 'config/single-tenant/main.yaml'

  271. 

  272.     def test_jobs_executed(self):

  273.         "Test that jobs are executed and a change is merged"

  274. 

  275.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  276.         A.addApproval('Code-Review', 2)

  277.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  278.         self.waitUntilSettled()

  279.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  280.                          'SUCCESS')

  281.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  282.                          'SUCCESS')

  283.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  284.                          'SUCCESS')

  285.         self.assertEqual(A.data['status'], 'MERGED')

  286.         self.assertEqual(A.reported, 2)

  287.         self.assertEqual(self.getJobFromHistory('project-test1').node,

  288.                          'label1')

  289.         self.assertEqual(self.getJobFromHistory('project-test2').node,

  290.                          'label1')

  291. 

  292.         # TODOv3(jeblair): we may want to report stats by tenant (also?).

  293.         # Per-driver

  294.         self.assertReportedStat('zuul.event.gerrit.comment-added', value='1',

  295.                                 kind='c')

  296.         # Per-driver per-connection

  297.         self.assertReportedStat('zuul.event.gerrit.gerrit.comment-added',

  298.                                 value='1', kind='c')

  299.         self.assertReportedStat(

  300.             'zuul.tenant.tenant-one.pipeline.gate.current_changes',

  301.             value='1', kind='g')

  302.         self.assertReportedStat(

  303.             'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'

  304.             'org_project.master.job.project-merge.SUCCESS', kind='ms')

  305.         self.assertReportedStat(

  306.             'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'

  307.             'org_project.master.job.project-merge.SUCCESS', value='1',

  308.             kind='c')

  309.         self.assertReportedStat(

  310.             'zuul.tenant.tenant-one.pipeline.gate.resident_time', kind='ms')

  311.         self.assertReportedStat(

  312.             'zuul.tenant.tenant-one.pipeline.gate.total_changes', value='1',

  313.             kind='c')

  314.         self.assertReportedStat(

  315.             'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'

  316.             'org_project.master.resident_time', kind='ms')

  317.         self.assertReportedStat(

  318.             'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'

  319.             'org_project.master.total_changes', value='1', kind='c')

  320.         exec_key = 'zuul.executor.%s' % self.executor_server.hostname.replace(

  321.             '.', '_')

  322.         self.assertReportedStat(exec_key + '.builds', value='1', kind='c')

  323.         self.assertReportedStat(exec_key + '.starting_builds', kind='g')

  324.         self.assertReportedStat(exec_key + '.starting_builds', kind='ms')

  325.         self.assertReportedStat(

  326.             'zuul.nodepool.requests.requested.total', value='1', kind='c')

  327.         self.assertReportedStat(

  328.             'zuul.nodepool.requests.requested.label.label1',

  329.             value='1', kind='c')

  330.         self.assertReportedStat(

  331.             'zuul.nodepool.requests.fulfilled.label.label1',

  332.             value='1', kind='c')

  333.         self.assertReportedStat(

  334.             'zuul.nodepool.requests.requested.size.1', value='1', kind='c')

  335.         self.assertReportedStat(

  336.             'zuul.nodepool.requests.fulfilled.size.1', value='1', kind='c')

  337.         self.assertReportedStat(

  338.             'zuul.nodepool.current_requests', value='1', kind='g')

  339.         self.assertReportedStat(

  340.             'zuul.executors.online', value='1', kind='g')

  341.         self.assertReportedStat(

  342.             'zuul.executors.accepting', value='1', kind='g')

  343.         self.assertReportedStat(

  344.             'zuul.mergers.online', value='1', kind='g')

  345. 

  346.         for build in self.history:

  347.             self.assertTrue(build.parameters['zuul']['voting'])

  348. 

  349.     def test_initial_pipeline_gauges(self):

  350.         "Test that each pipeline reported its length on start"

  351.         self.assertReportedStat('zuul.tenant.tenant-one.pipeline.gate.'

  352.                                 'current_changes',

  353.                                 value='0', kind='g')

  354.         self.assertReportedStat('zuul.tenant.tenant-one.pipeline.check.'

  355.                                 'current_changes',

  356.                                 value='0', kind='g')

  357. 

  358.     def test_job_branch(self):

  359.         "Test the correct variant of a job runs on a branch"

  360.         self.create_branch('org/project', 'stable')

  361.         self.fake_gerrit.addEvent(

  362.             self.fake_gerrit.getFakeBranchCreatedEvent(

  363.                 'org/project', 'stable'))

  364.         self.waitUntilSettled()

  365. 

  366.         A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A')

  367.         A.addApproval('Code-Review', 2)

  368.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  369.         self.waitUntilSettled()

  370.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  371.                          'SUCCESS')

  372.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  373.                          'SUCCESS')

  374.         self.assertEqual(A.data['status'], 'MERGED')

  375.         self.assertEqual(A.reported, 2,

  376.                          "A should report start and success")

  377.         self.assertIn('gate', A.messages[1],

  378.                       "A should transit gate")

  379.         self.assertEqual(self.getJobFromHistory('project-test1').node,

  380.                          'label2')

  381. 

  382.     @simple_layout('layouts/branch-deletion.yaml')

  383.     def test_branch_deletion(self):

  384.         "Test the correct variant of a job runs on a branch"

  385.         self._startMerger()

  386.         merger_gear = self.executor_server.merger_gearworker.gearman

  387.         for f in list(merger_gear.functions.keys()):

  388.             f = f.decode('utf8')

  389.             if f.startswith('merger:'):

  390.                 merger_gear.unRegisterFunction(f)

  391. 

  392.         self.create_branch('org/project', 'stable')

  393.         self.fake_gerrit.addEvent(

  394.             self.fake_gerrit.getFakeBranchCreatedEvent(

  395.                 'org/project', 'stable'))

  396.         self.waitUntilSettled()

  397.         A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A')

  398.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  399.         self.waitUntilSettled()

  400.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  401.                          'SUCCESS')

  402. 

  403.         self.delete_branch('org/project', 'stable')

  404.         path = os.path.join(self.executor_src_root, 'review.example.com')

  405.         shutil.rmtree(path)

  406. 

  407.         self.executor_server.hold_jobs_in_build = True

  408.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  409.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  410.         self.waitUntilSettled()

  411.         build = self.builds[0]

  412. 

  413.         # Make sure there is no stable branch in the checked out git repo.

  414.         pname = 'review.example.com/org/project'

  415.         work = build.getWorkspaceRepos([pname])

  416.         work = work[pname]

  417.         heads = set([str(x) for x in work.heads])

  418.         self.assertEqual(heads, set(['master']))

  419.         self.executor_server.hold_jobs_in_build = False

  420.         build.release()

  421.         self.waitUntilSettled()

  422.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  423.                          'SUCCESS')

  424. 

  425.     def test_parallel_changes(self):

  426.         "Test that changes are tested in parallel and merged in series"

  427. 

  428.         self.executor_server.hold_jobs_in_build = True

  429.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  430.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  431.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  432.         A.addApproval('Code-Review', 2)

  433.         B.addApproval('Code-Review', 2)

  434.         C.addApproval('Code-Review', 2)

  435. 

  436.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  437.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  438.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  439. 

  440.         self.waitUntilSettled()

  441.         self.assertEqual(len(self.builds), 1)

  442.         self.assertEqual(self.builds[0].name, 'project-merge')

  443.         self.assertTrue(self.builds[0].hasChanges(A))

  444. 

  445.         self.executor_server.release('.*-merge')

  446.         self.waitUntilSettled()

  447.         self.assertEqual(len(self.builds), 3)

  448.         self.assertEqual(self.builds[0].name, 'project-test1')

  449.         self.assertTrue(self.builds[0].hasChanges(A))

  450.         self.assertEqual(self.builds[1].name, 'project-test2')

  451.         self.assertTrue(self.builds[1].hasChanges(A))

  452.         self.assertEqual(self.builds[2].name, 'project-merge')

  453.         self.assertTrue(self.builds[2].hasChanges(A, B))

  454. 

  455.         self.executor_server.release('.*-merge')

  456.         self.waitUntilSettled()

  457.         self.assertEqual(len(self.builds), 5)

  458.         self.assertEqual(self.builds[0].name, 'project-test1')

  459.         self.assertTrue(self.builds[0].hasChanges(A))

  460.         self.assertEqual(self.builds[1].name, 'project-test2')

  461.         self.assertTrue(self.builds[1].hasChanges(A))

  462. 

  463.         self.assertEqual(self.builds[2].name, 'project-test1')

  464.         self.assertTrue(self.builds[2].hasChanges(A, B))

  465.         self.assertEqual(self.builds[3].name, 'project-test2')

  466.         self.assertTrue(self.builds[3].hasChanges(A, B))

  467. 

  468.         self.assertEqual(self.builds[4].name, 'project-merge')

  469.         self.assertTrue(self.builds[4].hasChanges(A, B, C))

  470. 

  471.         self.executor_server.release('.*-merge')

  472.         self.waitUntilSettled()

  473.         self.assertEqual(len(self.builds), 6)

  474.         self.assertEqual(self.builds[0].name, 'project-test1')

  475.         self.assertTrue(self.builds[0].hasChanges(A))

  476.         self.assertEqual(self.builds[1].name, 'project-test2')

  477.         self.assertTrue(self.builds[1].hasChanges(A))

  478. 

  479.         self.assertEqual(self.builds[2].name, 'project-test1')

  480.         self.assertTrue(self.builds[2].hasChanges(A, B))

  481.         self.assertEqual(self.builds[3].name, 'project-test2')

  482.         self.assertTrue(self.builds[3].hasChanges(A, B))

  483. 

  484.         self.assertEqual(self.builds[4].name, 'project-test1')

  485.         self.assertTrue(self.builds[4].hasChanges(A, B, C))

  486.         self.assertEqual(self.builds[5].name, 'project-test2')

  487.         self.assertTrue(self.builds[5].hasChanges(A, B, C))

  488. 

  489.         self.executor_server.hold_jobs_in_build = False

  490.         self.executor_server.release()

  491.         self.waitUntilSettled()

  492.         self.assertEqual(len(self.builds), 0)

  493. 

  494.         self.assertEqual(len(self.history), 9)

  495.         self.assertEqual(A.data['status'], 'MERGED')

  496.         self.assertEqual(B.data['status'], 'MERGED')

  497.         self.assertEqual(C.data['status'], 'MERGED')

  498.         self.assertEqual(A.reported, 2)

  499.         self.assertEqual(B.reported, 2)

  500.         self.assertEqual(C.reported, 2)

  501. 

  502.     def test_failed_changes(self):

  503.         "Test that a change behind a failed change is retested"

  504.         self.executor_server.hold_jobs_in_build = True

  505. 

  506.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  507.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  508.         A.addApproval('Code-Review', 2)

  509.         B.addApproval('Code-Review', 2)

  510. 

  511.         self.executor_server.failJob('project-test1', A)

  512. 

  513.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  514.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  515.         self.waitUntilSettled()

  516.         self.assertBuilds([dict(name='project-merge', changes='1,1')])

  517. 

  518.         self.executor_server.release('.*-merge')

  519.         self.waitUntilSettled()

  520.         # A/project-merge is complete

  521.         self.assertBuilds([

  522.             dict(name='project-test1', changes='1,1'),

  523.             dict(name='project-test2', changes='1,1'),

  524.             dict(name='project-merge', changes='1,1 2,1'),

  525.         ])

  526. 

  527.         self.executor_server.release('.*-merge')

  528.         self.waitUntilSettled()

  529.         # A/project-merge is complete

  530.         # B/project-merge is complete

  531.         self.assertBuilds([

  532.             dict(name='project-test1', changes='1,1'),

  533.             dict(name='project-test2', changes='1,1'),

  534.             dict(name='project-test1', changes='1,1 2,1'),

  535.             dict(name='project-test2', changes='1,1 2,1'),

  536.         ])

  537. 

  538.         # Release project-test1 for A which will fail.  This will

  539.         # abort both running B jobs and reexecute project-merge for B.

  540.         self.builds[0].release()

  541.         self.waitUntilSettled()

  542. 

  543.         self.orderedRelease()

  544.         self.assertHistory([

  545.             dict(name='project-merge', result='SUCCESS', changes='1,1'),

  546.             dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),

  547.             dict(name='project-test1', result='FAILURE', changes='1,1'),

  548.             dict(name='project-test1', result='ABORTED', changes='1,1 2,1'),

  549.             dict(name='project-test2', result='ABORTED', changes='1,1 2,1'),

  550.             dict(name='project-test2', result='SUCCESS', changes='1,1'),

  551.             dict(name='project-merge', result='SUCCESS', changes='2,1'),

  552.             dict(name='project-test1', result='SUCCESS', changes='2,1'),

  553.             dict(name='project-test2', result='SUCCESS', changes='2,1'),

  554.         ], ordered=False)

  555. 

  556.         self.assertEqual(A.data['status'], 'NEW')

  557.         self.assertEqual(B.data['status'], 'MERGED')

  558.         self.assertEqual(A.reported, 2)

  559.         self.assertEqual(B.reported, 2)

  560. 

  561.     def test_independent_queues(self):

  562.         "Test that changes end up in the right queues"

  563. 

  564.         self.executor_server.hold_jobs_in_build = True

  565.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  566.         B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')

  567.         C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')

  568.         A.addApproval('Code-Review', 2)

  569.         B.addApproval('Code-Review', 2)

  570.         C.addApproval('Code-Review', 2)

  571. 

  572.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  573.         self.waitUntilSettled()

  574. 

  575.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  576.         self.waitUntilSettled()

  577. 

  578.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  579.         self.waitUntilSettled()

  580. 

  581.         # There should be one merge job at the head of each queue running

  582.         self.assertBuilds([

  583.             dict(name='project-merge', changes='1,1'),

  584.             dict(name='project-merge', changes='2,1'),

  585.         ])

  586. 

  587.         # Release the current merge builds

  588.         self.builds[0].release()

  589.         self.waitUntilSettled()

  590.         self.builds[0].release()

  591.         self.waitUntilSettled()

  592.         # Release the merge job for project2 which is behind project1

  593.         self.executor_server.release('.*-merge')

  594.         self.waitUntilSettled()

  595. 

  596.         # All the test builds should be running:

  597.         self.assertBuilds([

  598.             dict(name='project-test1', changes='1,1'),

  599.             dict(name='project-test2', changes='1,1'),

  600.             dict(name='project-test1', changes='2,1'),

  601.             dict(name='project-test2', changes='2,1'),

  602.             dict(name='project1-project2-integration', changes='2,1'),

  603.             dict(name='project-test1', changes='2,1 3,1'),

  604.             dict(name='project-test2', changes='2,1 3,1'),

  605.             dict(name='project1-project2-integration', changes='2,1 3,1'),

  606.         ])

  607. 

  608.         self.orderedRelease()

  609.         self.assertHistory([

  610.             dict(name='project-merge', result='SUCCESS', changes='1,1'),

  611.             dict(name='project-merge', result='SUCCESS', changes='2,1'),

  612.             dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'),

  613.             dict(name='project-test1', result='SUCCESS', changes='1,1'),

  614.             dict(name='project-test2', result='SUCCESS', changes='1,1'),

  615.             dict(name='project-test1', result='SUCCESS', changes='2,1'),

  616.             dict(name='project-test2', result='SUCCESS', changes='2,1'),

  617.             dict(

  618.                 name='project1-project2-integration',

  619.                 result='SUCCESS',

  620.                 changes='2,1'),

  621.             dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'),

  622.             dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'),

  623.             dict(name='project1-project2-integration',

  624.                  result='SUCCESS',

  625.                  changes='2,1 3,1'),

  626.         ])

  627. 

  628.         self.assertEqual(A.data['status'], 'MERGED')

  629.         self.assertEqual(B.data['status'], 'MERGED')

  630.         self.assertEqual(C.data['status'], 'MERGED')

  631.         self.assertEqual(A.reported, 2)

  632.         self.assertEqual(B.reported, 2)

  633.         self.assertEqual(C.reported, 2)

  634. 

  635.     def test_failed_change_at_head(self):

  636.         "Test that if a change at the head fails, jobs behind it are canceled"

  637. 

  638.         self.executor_server.hold_jobs_in_build = True

  639.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  640.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  641.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  642.         A.addApproval('Code-Review', 2)

  643.         B.addApproval('Code-Review', 2)

  644.         C.addApproval('Code-Review', 2)

  645. 

  646.         self.executor_server.failJob('project-test1', A)

  647. 

  648.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  649.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  650.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  651. 

  652.         self.waitUntilSettled()

  653. 

  654.         self.assertBuilds([

  655.             dict(name='project-merge', changes='1,1'),

  656.         ])

  657. 

  658.         self.executor_server.release('.*-merge')

  659.         self.waitUntilSettled()

  660.         self.executor_server.release('.*-merge')

  661.         self.waitUntilSettled()

  662.         self.executor_server.release('.*-merge')

  663.         self.waitUntilSettled()

  664. 

  665.         self.assertBuilds([

  666.             dict(name='project-test1', changes='1,1'),

  667.             dict(name='project-test2', changes='1,1'),

  668.             dict(name='project-test1', changes='1,1 2,1'),

  669.             dict(name='project-test2', changes='1,1 2,1'),

  670.             dict(name='project-test1', changes='1,1 2,1 3,1'),

  671.             dict(name='project-test2', changes='1,1 2,1 3,1'),

  672.         ])

  673. 

  674.         self.release(self.builds[0])

  675.         self.waitUntilSettled()

  676. 

  677.         # project-test2, project-merge for B

  678.         self.assertBuilds([

  679.             dict(name='project-test2', changes='1,1'),

  680.             dict(name='project-merge', changes='2,1'),

  681.         ])

  682.         # Unordered history comparison because the aborts can finish

  683.         # in any order.

  684.         self.assertHistory([

  685.             dict(name='project-merge', result='SUCCESS',

  686.                  changes='1,1'),

  687.             dict(name='project-merge', result='SUCCESS',

  688.                  changes='1,1 2,1'),

  689.             dict(name='project-merge', result='SUCCESS',

  690.                  changes='1,1 2,1 3,1'),

  691.             dict(name='project-test1', result='FAILURE',

  692.                  changes='1,1'),

  693.             dict(name='project-test1', result='ABORTED',

  694.                  changes='1,1 2,1'),

  695.             dict(name='project-test2', result='ABORTED',

  696.                  changes='1,1 2,1'),

  697.             dict(name='project-test1', result='ABORTED',

  698.                  changes='1,1 2,1 3,1'),

  699.             dict(name='project-test2', result='ABORTED',

  700.                  changes='1,1 2,1 3,1'),

  701.         ], ordered=False)

  702. 

  703.         self.executor_server.release('.*-merge')

  704.         self.waitUntilSettled()

  705.         self.executor_server.release('.*-merge')

  706.         self.waitUntilSettled()

  707.         self.orderedRelease()

  708. 

  709.         self.assertBuilds([])

  710.         self.assertHistory([

  711.             dict(name='project-merge', result='SUCCESS',

  712.                  changes='1,1'),

  713.             dict(name='project-merge', result='SUCCESS',

  714.                  changes='1,1 2,1'),

  715.             dict(name='project-merge', result='SUCCESS',

  716.                  changes='1,1 2,1 3,1'),

  717.             dict(name='project-test1', result='FAILURE',

  718.                  changes='1,1'),

  719.             dict(name='project-test1', result='ABORTED',

  720.                  changes='1,1 2,1'),

  721.             dict(name='project-test2', result='ABORTED',

  722.                  changes='1,1 2,1'),

  723.             dict(name='project-test1', result='ABORTED',

  724.                  changes='1,1 2,1 3,1'),

  725.             dict(name='project-test2', result='ABORTED',

  726.                  changes='1,1 2,1 3,1'),

  727.             dict(name='project-merge', result='SUCCESS',

  728.                  changes='2,1'),

  729.             dict(name='project-merge', result='SUCCESS',

  730.                  changes='2,1 3,1'),

  731.             dict(name='project-test2', result='SUCCESS',

  732.                  changes='1,1'),

  733.             dict(name='project-test1', result='SUCCESS',

  734.                  changes='2,1'),

  735.             dict(name='project-test2', result='SUCCESS',

  736.                  changes='2,1'),

  737.             dict(name='project-test1', result='SUCCESS',

  738.                  changes='2,1 3,1'),

  739.             dict(name='project-test2', result='SUCCESS',

  740.                  changes='2,1 3,1'),

  741.         ], ordered=False)

  742. 

  743.         self.assertEqual(A.data['status'], 'NEW')

  744.         self.assertEqual(B.data['status'], 'MERGED')

  745.         self.assertEqual(C.data['status'], 'MERGED')

  746.         self.assertEqual(A.reported, 2)

  747.         self.assertEqual(B.reported, 2)

  748.         self.assertEqual(C.reported, 2)

  749. 

  750.     def test_failed_change_in_middle(self):

  751.         "Test a failed change in the middle of the queue"

  752. 

  753.         self.executor_server.hold_jobs_in_build = True

  754.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  755.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  756.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  757.         A.addApproval('Code-Review', 2)

  758.         B.addApproval('Code-Review', 2)

  759.         C.addApproval('Code-Review', 2)

  760. 

  761.         self.executor_server.failJob('project-test1', B)

  762. 

  763.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  764.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  765.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  766. 

  767.         self.waitUntilSettled()

  768. 

  769.         self.executor_server.release('.*-merge')

  770.         self.waitUntilSettled()

  771.         self.executor_server.release('.*-merge')

  772.         self.waitUntilSettled()

  773.         self.executor_server.release('.*-merge')

  774.         self.waitUntilSettled()

  775. 

  776.         self.assertEqual(len(self.builds), 6)

  777.         self.assertEqual(self.builds[0].name, 'project-test1')

  778.         self.assertEqual(self.builds[1].name, 'project-test2')

  779.         self.assertEqual(self.builds[2].name, 'project-test1')

  780.         self.assertEqual(self.builds[3].name, 'project-test2')

  781.         self.assertEqual(self.builds[4].name, 'project-test1')

  782.         self.assertEqual(self.builds[5].name, 'project-test2')

  783. 

  784.         self.release(self.builds[2])

  785.         self.waitUntilSettled()

  786. 

  787.         # project-test1 and project-test2 for A

  788.         # project-test2 for B

  789.         # project-merge for C (without B)

  790.         self.assertEqual(len(self.builds), 4)

  791.         self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 2)

  792. 

  793.         self.executor_server.release('.*-merge')

  794.         self.waitUntilSettled()

  795. 

  796.         # project-test1 and project-test2 for A

  797.         # project-test2 for B

  798.         # project-test1 and project-test2 for C

  799.         self.assertEqual(len(self.builds), 5)

  800. 

  801.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  802.         items = tenant.layout.pipelines['gate'].getAllItems()

  803.         builds = items[0].current_build_set.getBuilds()

  804.         self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)

  805.         self.assertEqual(self.countJobResults(builds, None), 2)

  806.         builds = items[1].current_build_set.getBuilds()

  807.         self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)

  808.         self.assertEqual(self.countJobResults(builds, 'FAILURE'), 1)

  809.         self.assertEqual(self.countJobResults(builds, None), 1)

  810.         builds = items[2].current_build_set.getBuilds()

  811.         self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)

  812.         self.assertEqual(self.countJobResults(builds, None), 2)

  813. 

  814.         self.executor_server.hold_jobs_in_build = False

  815.         self.executor_server.release()

  816.         self.waitUntilSettled()

  817. 

  818.         self.assertEqual(len(self.builds), 0)

  819.         self.assertEqual(len(self.history), 12)

  820.         self.assertEqual(A.data['status'], 'MERGED')

  821.         self.assertEqual(B.data['status'], 'NEW')

  822.         self.assertEqual(C.data['status'], 'MERGED')

  823.         self.assertEqual(A.reported, 2)

  824.         self.assertEqual(B.reported, 2)

  825.         self.assertEqual(C.reported, 2)

  826. 

  827.     def test_failed_change_at_head_with_queue(self):

  828.         "Test that if a change at the head fails, queued jobs are canceled"

  829. 

  830.         self.gearman_server.hold_jobs_in_queue = True

  831.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  832.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  833.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  834.         A.addApproval('Code-Review', 2)

  835.         B.addApproval('Code-Review', 2)

  836.         C.addApproval('Code-Review', 2)

  837. 

  838.         self.executor_server.failJob('project-test1', A)

  839. 

  840.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  841.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  842.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  843. 

  844.         self.waitUntilSettled()

  845.         queue = self.gearman_server.getQueue()

  846.         self.assertEqual(len(self.builds), 0)

  847.         self.assertEqual(len(queue), 1)

  848.         self.assertEqual(queue[0].name, b'executor:execute')

  849.         job_args = json.loads(queue[0].arguments.decode('utf8'))

  850.         self.assertEqual(job_args['job'], 'project-merge')

  851.         self.assertEqual(job_args['items'][0]['number'], '%d' % A.number)

  852. 

  853.         self.gearman_server.release('.*-merge')

  854.         self.waitUntilSettled()

  855.         self.gearman_server.release('.*-merge')

  856.         self.waitUntilSettled()

  857.         self.gearman_server.release('.*-merge')

  858.         self.waitUntilSettled()

  859.         queue = self.gearman_server.getQueue()

  860. 

  861.         self.assertEqual(len(self.builds), 0)

  862.         self.assertEqual(len(queue), 6)

  863. 

  864.         self.assertEqual(

  865.             json.loads(queue[0].arguments.decode('utf8'))['job'],

  866.             'project-test1')

  867.         self.assertEqual(

  868.             json.loads(queue[1].arguments.decode('utf8'))['job'],

  869.             'project-test2')

  870.         self.assertEqual(

  871.             json.loads(queue[2].arguments.decode('utf8'))['job'],

  872.             'project-test1')

  873.         self.assertEqual(

  874.             json.loads(queue[3].arguments.decode('utf8'))['job'],

  875.             'project-test2')

  876.         self.assertEqual(

  877.             json.loads(queue[4].arguments.decode('utf8'))['job'],

  878.             'project-test1')

  879.         self.assertEqual(

  880.             json.loads(queue[5].arguments.decode('utf8'))['job'],

  881.             'project-test2')

  882. 

  883.         self.release(queue[0])

  884.         self.waitUntilSettled()

  885. 

  886.         self.assertEqual(len(self.builds), 0)

  887.         queue = self.gearman_server.getQueue()

  888.         self.assertEqual(len(queue), 2)  # project-test2, project-merge for B

  889.         self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 0)

  890. 

  891.         self.gearman_server.hold_jobs_in_queue = False

  892.         self.gearman_server.release()

  893.         self.waitUntilSettled()

  894. 

  895.         self.assertEqual(len(self.builds), 0)

  896.         self.assertEqual(len(self.history), 11)

  897.         self.assertEqual(A.data['status'], 'NEW')

  898.         self.assertEqual(B.data['status'], 'MERGED')

  899.         self.assertEqual(C.data['status'], 'MERGED')

  900.         self.assertEqual(A.reported, 2)

  901.         self.assertEqual(B.reported, 2)

  902.         self.assertEqual(C.reported, 2)

  903. 

  904.     def _test_time_database(self, iteration):

  905.         self.executor_server.hold_jobs_in_build = True

  906.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  907.         A.addApproval('Code-Review', 2)

  908.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  909.         self.waitUntilSettled()

  910.         time.sleep(2)

  911. 

  912.         data = json.loads(self.scheds.first.sched

  913.                           .formatStatusJSON('tenant-one'))

  914.         found_job = None

  915.         for pipeline in data['pipelines']:

  916.             if pipeline['name'] != 'gate':

  917.                 continue

  918.             for queue in pipeline['change_queues']:

  919.                 for head in queue['heads']:

  920.                     for item in head:

  921.                         for job in item['jobs']:

  922.                             if job['name'] == 'project-merge':

  923.                                 found_job = job

  924.                                 break

  925. 

  926.         self.assertIsNotNone(found_job)

  927.         if iteration == 1:

  928.             self.assertIsNotNone(found_job['estimated_time'])

  929.             self.assertIsNone(found_job['remaining_time'])

  930.         else:

  931.             self.assertIsNotNone(found_job['estimated_time'])

  932.             self.assertTrue(found_job['estimated_time'] >= 2)

  933.             self.assertIsNotNone(found_job['remaining_time'])

  934. 

  935.         self.executor_server.hold_jobs_in_build = False

  936.         self.executor_server.release()

  937.         self.waitUntilSettled()

  938. 

  939.     def test_time_database(self):

  940.         "Test the time database"

  941. 

  942.         self._test_time_database(1)

  943.         self._test_time_database(2)

  944. 

  945.     def test_two_failed_changes_at_head(self):

  946.         "Test that changes are reparented correctly if 2 fail at head"

  947. 

  948.         self.executor_server.hold_jobs_in_build = True

  949.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  950.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  951.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  952.         A.addApproval('Code-Review', 2)

  953.         B.addApproval('Code-Review', 2)

  954.         C.addApproval('Code-Review', 2)

  955. 

  956.         self.executor_server.failJob('project-test1', A)

  957.         self.executor_server.failJob('project-test1', B)

  958. 

  959.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  960.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  961.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  962.         self.waitUntilSettled()

  963. 

  964.         self.executor_server.release('.*-merge')

  965.         self.waitUntilSettled()

  966.         self.executor_server.release('.*-merge')

  967.         self.waitUntilSettled()

  968.         self.executor_server.release('.*-merge')

  969.         self.waitUntilSettled()

  970. 

  971.         self.assertEqual(len(self.builds), 6)

  972.         self.assertEqual(self.builds[0].name, 'project-test1')

  973.         self.assertEqual(self.builds[1].name, 'project-test2')

  974.         self.assertEqual(self.builds[2].name, 'project-test1')

  975.         self.assertEqual(self.builds[3].name, 'project-test2')

  976.         self.assertEqual(self.builds[4].name, 'project-test1')

  977.         self.assertEqual(self.builds[5].name, 'project-test2')

  978. 

  979.         self.assertTrue(self.builds[0].hasChanges(A))

  980.         self.assertTrue(self.builds[2].hasChanges(A))

  981.         self.assertTrue(self.builds[2].hasChanges(B))

  982.         self.assertTrue(self.builds[4].hasChanges(A))

  983.         self.assertTrue(self.builds[4].hasChanges(B))

  984.         self.assertTrue(self.builds[4].hasChanges(C))

  985. 

  986.         # Fail change B first

  987.         self.release(self.builds[2])

  988.         self.waitUntilSettled()

  989. 

  990.         # restart of C after B failure

  991.         self.executor_server.release('.*-merge')

  992.         self.waitUntilSettled()

  993. 

  994.         self.assertEqual(len(self.builds), 5)

  995.         self.assertEqual(self.builds[0].name, 'project-test1')

  996.         self.assertEqual(self.builds[1].name, 'project-test2')

  997.         self.assertEqual(self.builds[2].name, 'project-test2')

  998.         self.assertEqual(self.builds[3].name, 'project-test1')

  999.         self.assertEqual(self.builds[4].name, 'project-test2')

  1000. 

  1001.         self.assertTrue(self.builds[1].hasChanges(A))

  1002.         self.assertTrue(self.builds[2].hasChanges(A))

  1003.         self.assertTrue(self.builds[2].hasChanges(B))

  1004.         self.assertTrue(self.builds[4].hasChanges(A))

  1005.         self.assertFalse(self.builds[4].hasChanges(B))

  1006.         self.assertTrue(self.builds[4].hasChanges(C))

  1007. 

  1008.         # Finish running all passing jobs for change A

  1009.         self.release(self.builds[1])

  1010.         self.waitUntilSettled()

  1011.         # Fail and report change A

  1012.         self.release(self.builds[0])

  1013.         self.waitUntilSettled()

  1014. 

  1015.         # restart of B,C after A failure

  1016.         self.executor_server.release('.*-merge')

  1017.         self.waitUntilSettled()

  1018.         self.executor_server.release('.*-merge')

  1019.         self.waitUntilSettled()

  1020. 

  1021.         self.assertEqual(len(self.builds), 4)

  1022.         self.assertEqual(self.builds[0].name, 'project-test1')  # B

  1023.         self.assertEqual(self.builds[1].name, 'project-test2')  # B

  1024.         self.assertEqual(self.builds[2].name, 'project-test1')  # C

  1025.         self.assertEqual(self.builds[3].name, 'project-test2')  # C

  1026. 

  1027.         self.assertFalse(self.builds[1].hasChanges(A))

  1028.         self.assertTrue(self.builds[1].hasChanges(B))

  1029.         self.assertFalse(self.builds[1].hasChanges(C))

  1030. 

  1031.         self.assertFalse(self.builds[2].hasChanges(A))

  1032.         # After A failed and B and C restarted, B should be back in

  1033.         # C's tests because it has not failed yet.

  1034.         self.assertTrue(self.builds[2].hasChanges(B))

  1035.         self.assertTrue(self.builds[2].hasChanges(C))

  1036. 

  1037.         self.executor_server.hold_jobs_in_build = False

  1038.         self.executor_server.release()

  1039.         self.waitUntilSettled()

  1040. 

  1041.         self.assertEqual(len(self.builds), 0)

  1042.         self.assertEqual(len(self.history), 21)

  1043.         self.assertEqual(A.data['status'], 'NEW')

  1044.         self.assertEqual(B.data['status'], 'NEW')

  1045.         self.assertEqual(C.data['status'], 'MERGED')

  1046.         self.assertEqual(A.reported, 2)

  1047.         self.assertEqual(B.reported, 2)

  1048.         self.assertEqual(C.reported, 2)

  1049. 

  1050.     def test_patch_order(self):

  1051.         "Test that dependent patches are tested in the right order"

  1052.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1053.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  1054.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  1055.         A.addApproval('Code-Review', 2)

  1056.         B.addApproval('Code-Review', 2)

  1057.         C.addApproval('Code-Review', 2)

  1058. 

  1059.         M2 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M2')

  1060.         M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')

  1061.         M2.setMerged()

  1062.         M1.setMerged()

  1063. 

  1064.         # C -> B -> A -> M1 -> M2

  1065.         # M2 is here to make sure it is never queried.  If it is, it

  1066.         # means zuul is walking down the entire history of merged

  1067.         # changes.

  1068. 

  1069.         C.setDependsOn(B, 1)

  1070.         B.setDependsOn(A, 1)

  1071.         A.setDependsOn(M1, 1)

  1072.         M1.setDependsOn(M2, 1)

  1073. 

  1074.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  1075. 

  1076.         self.waitUntilSettled()

  1077. 

  1078.         self.assertEqual(A.data['status'], 'NEW')

  1079.         self.assertEqual(B.data['status'], 'NEW')

  1080.         self.assertEqual(C.data['status'], 'NEW')

  1081. 

  1082.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  1083.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  1084. 

  1085.         self.waitUntilSettled()

  1086.         self.assertEqual(M2.queried, 0)

  1087.         self.assertEqual(A.data['status'], 'MERGED')

  1088.         self.assertEqual(B.data['status'], 'MERGED')

  1089.         self.assertEqual(C.data['status'], 'MERGED')

  1090.         self.assertEqual(A.reported, 2)

  1091.         self.assertEqual(B.reported, 2)

  1092.         self.assertEqual(C.reported, 2)

  1093. 

  1094.     def test_needed_changes_enqueue(self):

  1095.         "Test that a needed change is enqueued ahead"

  1096.         #          A      Given a git tree like this, if we enqueue

  1097.         #         / \     change C, we should walk up and down the tree

  1098.         #        B   G    and enqueue changes in the order ABCDEFG.

  1099.         #       /|\       This is also the order that you would get if

  1100.         #     *C E F      you enqueued changes in the order ABCDEFG, so

  1101.         #     /           the ordering is stable across re-enqueue events.

  1102.         #    D

  1103. 

  1104.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1105.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  1106.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  1107.         D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')

  1108.         E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')

  1109.         F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')

  1110.         G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')

  1111.         B.setDependsOn(A, 1)

  1112.         C.setDependsOn(B, 1)

  1113.         D.setDependsOn(C, 1)

  1114.         E.setDependsOn(B, 1)

  1115.         F.setDependsOn(B, 1)

  1116.         G.setDependsOn(A, 1)

  1117. 

  1118.         A.addApproval('Code-Review', 2)

  1119.         B.addApproval('Code-Review', 2)

  1120.         C.addApproval('Code-Review', 2)

  1121.         D.addApproval('Code-Review', 2)

  1122.         E.addApproval('Code-Review', 2)

  1123.         F.addApproval('Code-Review', 2)

  1124.         G.addApproval('Code-Review', 2)

  1125.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  1126. 

  1127.         self.waitUntilSettled()

  1128. 

  1129.         self.assertEqual(A.data['status'], 'NEW')

  1130.         self.assertEqual(B.data['status'], 'NEW')

  1131.         self.assertEqual(C.data['status'], 'NEW')

  1132.         self.assertEqual(D.data['status'], 'NEW')

  1133.         self.assertEqual(E.data['status'], 'NEW')

  1134.         self.assertEqual(F.data['status'], 'NEW')

  1135.         self.assertEqual(G.data['status'], 'NEW')

  1136. 

  1137.         # We're about to add approvals to changes without adding the

  1138.         # triggering events to Zuul, so that we can be sure that it is

  1139.         # enqueing the changes based on dependencies, not because of

  1140.         # triggering events.  Since it will have the changes cached

  1141.         # already (without approvals), we need to clear the cache

  1142.         # first.

  1143.         for connection in self.scheds.first.connections.connections.values():

  1144.             connection.maintainCache([])

  1145. 

  1146.         self.executor_server.hold_jobs_in_build = True

  1147.         A.addApproval('Approved', 1)

  1148.         B.addApproval('Approved', 1)

  1149.         D.addApproval('Approved', 1)

  1150.         E.addApproval('Approved', 1)

  1151.         F.addApproval('Approved', 1)

  1152.         G.addApproval('Approved', 1)

  1153.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  1154. 

  1155.         for x in range(8):

  1156.             self.executor_server.release('.*-merge')

  1157.             self.waitUntilSettled()

  1158.         self.executor_server.hold_jobs_in_build = False

  1159.         self.executor_server.release()

  1160.         self.waitUntilSettled()

  1161. 

  1162.         self.assertEqual(A.data['status'], 'MERGED')

  1163.         self.assertEqual(B.data['status'], 'MERGED')

  1164.         self.assertEqual(C.data['status'], 'MERGED')

  1165.         self.assertEqual(D.data['status'], 'MERGED')

  1166.         self.assertEqual(E.data['status'], 'MERGED')

  1167.         self.assertEqual(F.data['status'], 'MERGED')

  1168.         self.assertEqual(G.data['status'], 'MERGED')

  1169.         self.assertEqual(A.reported, 2)

  1170.         self.assertEqual(B.reported, 2)

  1171.         self.assertEqual(C.reported, 2)

  1172.         self.assertEqual(D.reported, 2)

  1173.         self.assertEqual(E.reported, 2)

  1174.         self.assertEqual(F.reported, 2)

  1175.         self.assertEqual(G.reported, 2)

  1176.         self.assertEqual(self.history[6].changes,

  1177.                          '1,1 2,1 3,1 4,1 5,1 6,1 7,1')

  1178. 

  1179.     def test_source_cache(self):

  1180.         "Test that the source cache operates correctly"

  1181.         self.executor_server.hold_jobs_in_build = True

  1182. 

  1183.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1184.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  1185.         X = self.fake_gerrit.addFakeChange('org/project', 'master', 'X')

  1186.         A.addApproval('Code-Review', 2)

  1187.         B.addApproval('Code-Review', 2)

  1188. 

  1189.         M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')

  1190.         M1.setMerged()

  1191. 

  1192.         B.setDependsOn(A, 1)

  1193.         A.setDependsOn(M1, 1)

  1194. 

  1195.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  1196.         self.fake_gerrit.addEvent(X.getPatchsetCreatedEvent(1))

  1197. 

  1198.         self.waitUntilSettled()

  1199. 

  1200.         for build in self.builds:

  1201.             if build.pipeline == 'check':

  1202.                 build.release()

  1203.         self.waitUntilSettled()

  1204.         for build in self.builds:

  1205.             if build.pipeline == 'check':

  1206.                 build.release()

  1207.         self.waitUntilSettled()

  1208. 

  1209.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  1210.         self.waitUntilSettled()

  1211. 

  1212.         self.log.debug("len %s" % self.fake_gerrit._change_cache.keys())

  1213.         # there should still be changes in the cache

  1214.         self.assertNotEqual(len(self.fake_gerrit._change_cache.keys()), 0)

  1215. 

  1216.         self.executor_server.hold_jobs_in_build = False

  1217.         self.executor_server.release()

  1218.         self.waitUntilSettled()

  1219. 

  1220.         self.assertEqual(A.data['status'], 'MERGED')

  1221.         self.assertEqual(B.data['status'], 'MERGED')

  1222.         self.assertEqual(A.queried, 2)  # Initial and isMerged

  1223.         self.assertEqual(B.queried, 3)  # Initial A, refresh from B, isMerged

  1224. 

  1225.     def test_can_merge(self):

  1226.         "Test whether a change is ready to merge"

  1227.         # TODO: move to test_gerrit (this is a unit test!)

  1228.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1229.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  1230.         (trusted, project) = tenant.getProject('org/project')

  1231.         source = project.source

  1232. 

  1233.         # TODO(pabelanger): As we add more source / trigger APIs we should make

  1234.         # it easier for users to create events for testing.

  1235.         event = zuul.model.TriggerEvent()

  1236.         event.trigger_name = 'gerrit'

  1237.         event.change_number = '1'

  1238.         event.patch_number = '2'

  1239. 

  1240.         a = source.getChange(event)

  1241.         mgr = tenant.layout.pipelines['gate'].manager

  1242.         self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))

  1243. 

  1244.         A.addApproval('Code-Review', 2)

  1245.         a = source.getChange(event, refresh=True)

  1246.         self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))

  1247. 

  1248.         A.addApproval('Approved', 1)

  1249.         a = source.getChange(event, refresh=True)

  1250.         self.assertTrue(source.canMerge(a, mgr.getSubmitAllowNeeds()))

  1251. 

  1252.     def test_project_merge_conflict(self):

  1253.         "Test that gate merge conflicts are handled properly"

  1254. 

  1255.         self.gearman_server.hold_jobs_in_queue = True

  1256.         A = self.fake_gerrit.addFakeChange('org/project',

  1257.                                            'master', 'A',

  1258.                                            files={'conflict': 'foo'})

  1259.         B = self.fake_gerrit.addFakeChange('org/project',

  1260.                                            'master', 'B',

  1261.                                            files={'conflict': 'bar'})

  1262.         C = self.fake_gerrit.addFakeChange('org/project',

  1263.                                            'master', 'C')

  1264.         A.addApproval('Code-Review', 2)

  1265.         B.addApproval('Code-Review', 2)

  1266.         C.addApproval('Code-Review', 2)

  1267.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  1268.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  1269.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  1270.         self.waitUntilSettled()

  1271. 

  1272.         self.assertEqual(A.reported, 1)

  1273.         self.assertEqual(C.reported, 1)

  1274. 

  1275.         self.gearman_server.release('project-merge')

  1276.         self.waitUntilSettled()

  1277.         self.gearman_server.release('project-merge')

  1278.         self.waitUntilSettled()

  1279.         self.gearman_server.release('project-merge')

  1280.         self.waitUntilSettled()

  1281. 

  1282.         self.gearman_server.hold_jobs_in_queue = False

  1283.         self.gearman_server.release()

  1284.         self.waitUntilSettled()

  1285. 

  1286.         self.assertEqual(A.data['status'], 'MERGED')

  1287.         self.assertEqual(B.data['status'], 'NEW')

  1288.         self.assertEqual(C.data['status'], 'MERGED')

  1289.         self.assertEqual(A.reported, 2)

  1290.         self.assertIn('Merge Failed', B.messages[-1])

  1291.         self.assertEqual(C.reported, 2)

  1292. 

  1293.         self.assertHistory([

  1294.             dict(name='project-merge', result='SUCCESS', changes='1,1'),

  1295.             dict(name='project-test1', result='SUCCESS', changes='1,1'),

  1296.             dict(name='project-test2', result='SUCCESS', changes='1,1'),

  1297.             dict(name='project-merge', result='SUCCESS', changes='1,1 3,1'),

  1298.             dict(name='project-test1', result='SUCCESS', changes='1,1 3,1'),

  1299.             dict(name='project-test2', result='SUCCESS', changes='1,1 3,1'),

  1300.         ], ordered=False)

  1301. 

  1302.     def test_delayed_merge_conflict(self):

  1303.         "Test that delayed check merge conflicts are handled properly"

  1304. 

  1305.         # Hold jobs in the gearman queue so that we can test whether

  1306.         # the executor sucesfully merges a change based on an old

  1307.         # repo state (frozen by the scheduler) which would otherwise

  1308.         # conflict.

  1309.         self.gearman_server.hold_jobs_in_queue = True

  1310.         A = self.fake_gerrit.addFakeChange('org/project',

  1311.                                            'master', 'A',

  1312.                                            files={'conflict': 'foo'})

  1313.         B = self.fake_gerrit.addFakeChange('org/project',

  1314.                                            'master', 'B',

  1315.                                            files={'conflict': 'bar'})

  1316.         C = self.fake_gerrit.addFakeChange('org/project',

  1317.                                            'master', 'C')

  1318.         C.setDependsOn(B, 1)

  1319. 

  1320.         # A enters the gate queue; B and C enter the check queue

  1321.         A.addApproval('Code-Review', 2)

  1322.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  1323.         self.waitUntilSettled()

  1324.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  1325.         self.waitUntilSettled()

  1326.         self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

  1327.         self.waitUntilSettled()

  1328. 

  1329.         self.assertEqual(A.reported, 1)

  1330.         self.assertEqual(B.reported, 0)  # Check does not report start

  1331.         self.assertEqual(C.reported, 0)  # Check does not report start

  1332. 

  1333.         # A merges while B and C are queued in check

  1334.         # Release A project-merge

  1335.         queue = self.gearman_server.getQueue()

  1336.         self.release(queue[0])

  1337.         self.waitUntilSettled()

  1338. 

  1339.         # Release A project-test*

  1340.         # gate has higher precedence, so A's test jobs are added in

  1341.         # front of the merge jobs for B and C

  1342.         queue = self.gearman_server.getQueue()

  1343.         self.release(queue[0])

  1344.         self.release(queue[1])

  1345.         self.waitUntilSettled()

  1346. 

  1347.         self.assertEqual(A.data['status'], 'MERGED')

  1348.         self.assertEqual(B.data['status'], 'NEW')

  1349.         self.assertEqual(C.data['status'], 'NEW')

  1350.         self.assertEqual(A.reported, 2)

  1351.         self.assertEqual(B.reported, 0)

  1352.         self.assertEqual(C.reported, 0)

  1353.         self.assertHistory([

  1354.             dict(name='project-merge', result='SUCCESS', changes='1,1'),

  1355.             dict(name='project-test1', result='SUCCESS', changes='1,1'),

  1356.             dict(name='project-test2', result='SUCCESS', changes='1,1'),

  1357.         ], ordered=False)

  1358. 

  1359.         # B and C report merge conflicts

  1360.         # Release B project-merge

  1361.         queue = self.gearman_server.getQueue()

  1362.         self.release(queue[0])

  1363.         self.waitUntilSettled()

  1364. 

  1365.         # Release C

  1366.         self.gearman_server.hold_jobs_in_queue = False

  1367.         self.gearman_server.release()

  1368.         self.waitUntilSettled()

  1369. 

  1370.         self.assertEqual(A.data['status'], 'MERGED')

  1371.         self.assertEqual(B.data['status'], 'NEW')

  1372.         self.assertEqual(C.data['status'], 'NEW')

  1373.         self.assertEqual(A.reported, 2)

  1374.         self.assertEqual(B.reported, 1)

  1375.         self.assertEqual(C.reported, 1)

  1376. 

  1377.         self.assertHistory([

  1378.             dict(name='project-merge', result='SUCCESS', changes='1,1'),

  1379.             dict(name='project-test1', result='SUCCESS', changes='1,1'),

  1380.             dict(name='project-test2', result='SUCCESS', changes='1,1'),

  1381.             dict(name='project-merge', result='SUCCESS', changes='2,1'),

  1382.             dict(name='project-test1', result='SUCCESS', changes='2,1'),

  1383.             dict(name='project-test2', result='SUCCESS', changes='2,1'),

  1384.             dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'),

  1385.             dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'),

  1386.             dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'),

  1387.         ], ordered=False)

  1388. 

  1389.     def test_post(self):

  1390.         "Test that post jobs run"

  1391.         p = "review.example.com/org/project"

  1392.         upstream = self.getUpstreamRepos([p])

  1393.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1394.         A.setMerged()

  1395.         A_commit = str(upstream[p].commit('master'))

  1396.         self.log.debug("A commit: %s" % A_commit)

  1397. 

  1398.         e = {

  1399.             "type": "ref-updated",

  1400.             "submitter": {

  1401.                 "name": "User Name",

  1402.             },

  1403.             "refUpdate": {

  1404.                 "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",

  1405.                 "newRev": A_commit,

  1406.                 "refName": "master",

  1407.                 "project": "org/project",

  1408.             }

  1409.         }

  1410.         self.fake_gerrit.addEvent(e)

  1411.         self.waitUntilSettled()

  1412. 

  1413.         job_names = [x.name for x in self.history]

  1414.         self.assertEqual(len(self.history), 1)

  1415.         self.assertIn('project-post', job_names)

  1416. 

  1417.     def test_post_ignore_deletes(self):

  1418.         "Test that deleting refs does not trigger post jobs"

  1419. 

  1420.         e = {

  1421.             "type": "ref-updated",

  1422.             "submitter": {

  1423.                 "name": "User Name",

  1424.             },

  1425.             "refUpdate": {

  1426.                 "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",

  1427.                 "newRev": "0000000000000000000000000000000000000000",

  1428.                 "refName": "master",

  1429.                 "project": "org/project",

  1430.             }

  1431.         }

  1432.         self.fake_gerrit.addEvent(e)

  1433.         self.waitUntilSettled()

  1434. 

  1435.         job_names = [x.name for x in self.history]

  1436.         self.assertEqual(len(self.history), 0)

  1437.         self.assertNotIn('project-post', job_names)

  1438. 

  1439.     @simple_layout('layouts/dont-ignore-ref-deletes.yaml')

  1440.     def test_post_ignore_deletes_negative(self):

  1441.         "Test that deleting refs does trigger post jobs"

  1442.         e = {

  1443.             "type": "ref-updated",

  1444.             "submitter": {

  1445.                 "name": "User Name",

  1446.             },

  1447.             "refUpdate": {

  1448.                 "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",

  1449.                 "newRev": "0000000000000000000000000000000000000000",

  1450.                 "refName": "testbranch",

  1451.                 "project": "org/project",

  1452.             }

  1453.         }

  1454.         self.fake_gerrit.addEvent(e)

  1455.         self.waitUntilSettled()

  1456. 

  1457.         job_names = [x.name for x in self.history]

  1458.         self.assertEqual(len(self.history), 1)

  1459.         self.assertIn('project-post', job_names)

  1460. 

  1461.     @skip("Disabled for early v3 development")

  1462.     def test_build_configuration_branch_interaction(self):

  1463.         "Test that switching between branches works"

  1464.         self.test_build_configuration()

  1465.         self.test_build_configuration_branch()

  1466.         # C has been merged, undo that

  1467.         path = os.path.join(self.upstream_root, "org/project")

  1468.         repo = git.Repo(path)

  1469.         repo.heads.master.commit = repo.commit('init')

  1470.         self.test_build_configuration()

  1471. 

  1472.     def test_dependent_changes_rebase(self):

  1473.         # Test that no errors occur when we walk a dependency tree

  1474.         # with an unused leaf node due to a rebase.

  1475.         # Start by constructing: C -> B -> A

  1476.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1477.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  1478.         B.setDependsOn(A, 1)

  1479. 

  1480.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  1481.         C.setDependsOn(B, 1)

  1482. 

  1483.         # Then rebase to form: D -> C -> A

  1484.         C.addPatchset()  # C,2

  1485.         C.setDependsOn(A, 1)

  1486. 

  1487.         D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')

  1488.         D.setDependsOn(C, 2)

  1489. 

  1490.         # Walk the entire tree

  1491.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  1492.         self.waitUntilSettled()

  1493.         self.assertEqual(len(self.history), 3)

  1494. 

  1495.         # Verify that walking just part of the tree still works

  1496.         self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))

  1497.         self.waitUntilSettled()

  1498.         self.assertEqual(len(self.history), 6)

  1499. 

  1500.     def test_dependent_changes_dequeue(self):

  1501.         "Test that dependent patches are not needlessly tested"

  1502. 

  1503.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1504.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  1505.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  1506.         A.addApproval('Code-Review', 2)

  1507.         B.addApproval('Code-Review', 2)

  1508.         C.addApproval('Code-Review', 2)

  1509. 

  1510.         M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')

  1511.         M1.setMerged()

  1512. 

  1513.         # C -> B -> A -> M1

  1514. 

  1515.         C.setDependsOn(B, 1)

  1516.         B.setDependsOn(A, 1)

  1517.         A.setDependsOn(M1, 1)

  1518. 

  1519.         self.executor_server.failJob('project-merge', A)

  1520. 

  1521.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  1522.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  1523.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  1524. 

  1525.         self.waitUntilSettled()

  1526. 

  1527.         self.assertEqual(A.data['status'], 'NEW')

  1528.         self.assertEqual(A.reported, 2)

  1529.         self.assertEqual(B.data['status'], 'NEW')

  1530.         self.assertEqual(B.reported, 2)

  1531.         self.assertEqual(C.data['status'], 'NEW')

  1532.         self.assertIn('This change depends on a change that failed to merge.',

  1533.                       C.messages[-1])

  1534.         self.assertEqual(len(self.history), 1)

  1535. 

  1536.     def test_failing_dependent_changes(self):

  1537.         "Test that failing dependent patches are taken out of stream"

  1538.         self.executor_server.hold_jobs_in_build = True

  1539.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1540.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  1541.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  1542.         D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')

  1543.         E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')

  1544.         A.addApproval('Code-Review', 2)

  1545.         B.addApproval('Code-Review', 2)

  1546.         C.addApproval('Code-Review', 2)

  1547.         D.addApproval('Code-Review', 2)

  1548.         E.addApproval('Code-Review', 2)

  1549. 

  1550.         # E, D -> C -> B, A

  1551. 

  1552.         D.setDependsOn(C, 1)

  1553.         C.setDependsOn(B, 1)

  1554. 

  1555.         self.executor_server.failJob('project-test1', B)

  1556. 

  1557.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  1558.         self.fake_gerrit.addEvent(D.addApproval('Approved', 1))

  1559.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  1560.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  1561.         self.fake_gerrit.addEvent(E.addApproval('Approved', 1))

  1562. 

  1563.         self.waitUntilSettled()

  1564.         self.executor_server.release('.*-merge')

  1565.         self.waitUntilSettled()

  1566.         self.executor_server.release('.*-merge')

  1567.         self.waitUntilSettled()

  1568.         self.executor_server.release('.*-merge')

  1569.         self.waitUntilSettled()

  1570.         self.executor_server.release('.*-merge')

  1571.         self.waitUntilSettled()

  1572.         self.executor_server.release('.*-merge')

  1573.         self.waitUntilSettled()

  1574. 

  1575.         self.executor_server.hold_jobs_in_build = False

  1576.         for build in self.builds:

  1577.             if build.parameters['zuul']['change'] != '1':

  1578.                 build.release()

  1579.                 self.waitUntilSettled()

  1580. 

  1581.         self.executor_server.release()

  1582.         self.waitUntilSettled()

  1583. 

  1584.         self.assertEqual(A.data['status'], 'MERGED')

  1585.         self.assertEqual(A.reported, 2)

  1586.         self.assertIn('Build succeeded', A.messages[1])

  1587.         self.assertEqual(B.data['status'], 'NEW')

  1588.         self.assertEqual(B.reported, 2)

  1589.         self.assertIn('Build failed', B.messages[1])

  1590.         self.assertEqual(C.data['status'], 'NEW')

  1591.         self.assertEqual(C.reported, 2)

  1592.         self.assertIn('depends on a change', C.messages[1])

  1593.         self.assertEqual(D.data['status'], 'NEW')

  1594.         self.assertEqual(D.reported, 2)

  1595.         self.assertIn('depends on a change', D.messages[1])

  1596.         self.assertEqual(E.data['status'], 'MERGED')

  1597.         self.assertEqual(E.reported, 2)

  1598.         self.assertIn('Build succeeded', E.messages[1])

  1599.         self.assertEqual(len(self.history), 18)

  1600. 

  1601.     def test_head_is_dequeued_once(self):

  1602.         "Test that if a change at the head fails it is dequeued only once"

  1603.         # If it's dequeued more than once, we should see extra

  1604.         # aborted jobs.

  1605. 

  1606.         self.executor_server.hold_jobs_in_build = True

  1607.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1608.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  1609.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  1610.         A.addApproval('Code-Review', 2)

  1611.         B.addApproval('Code-Review', 2)

  1612.         C.addApproval('Code-Review', 2)

  1613. 

  1614.         self.executor_server.failJob('project-test1', A)

  1615.         self.executor_server.failJob('project-test2', A)

  1616. 

  1617.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  1618.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  1619.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  1620. 

  1621.         self.waitUntilSettled()

  1622. 

  1623.         self.assertEqual(len(self.builds), 1)

  1624.         self.assertEqual(self.builds[0].name, 'project-merge')

  1625.         self.assertTrue(self.builds[0].hasChanges(A))

  1626. 

  1627.         self.executor_server.release('.*-merge')

  1628.         self.waitUntilSettled()

  1629.         self.executor_server.release('.*-merge')

  1630.         self.waitUntilSettled()

  1631.         self.executor_server.release('.*-merge')

  1632.         self.waitUntilSettled()

  1633. 

  1634.         self.assertEqual(len(self.builds), 6)

  1635.         self.assertEqual(self.builds[0].name, 'project-test1')

  1636.         self.assertEqual(self.builds[1].name, 'project-test2')

  1637.         self.assertEqual(self.builds[2].name, 'project-test1')

  1638.         self.assertEqual(self.builds[3].name, 'project-test2')

  1639.         self.assertEqual(self.builds[4].name, 'project-test1')

  1640.         self.assertEqual(self.builds[5].name, 'project-test2')

  1641. 

  1642.         self.release(self.builds[0])

  1643.         self.waitUntilSettled()

  1644. 

  1645.         self.assertEqual(len(self.builds), 2)  # test2, merge for B

  1646.         self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)

  1647. 

  1648.         self.executor_server.hold_jobs_in_build = False

  1649.         self.executor_server.release()

  1650.         self.waitUntilSettled()

  1651. 

  1652.         self.assertEqual(len(self.builds), 0)

  1653.         self.assertEqual(len(self.history), 15)

  1654. 

  1655.         self.assertEqual(A.data['status'], 'NEW')

  1656.         self.assertEqual(B.data['status'], 'MERGED')

  1657.         self.assertEqual(C.data['status'], 'MERGED')

  1658.         self.assertEqual(A.reported, 2)

  1659.         self.assertEqual(B.reported, 2)

  1660.         self.assertEqual(C.reported, 2)

  1661. 

  1662.     @simple_layout('layouts/nonvoting-job-approval.yaml')

  1663.     def test_nonvoting_job_approval(self):

  1664.         "Test that non-voting jobs don't vote but leave approval"

  1665. 

  1666.         A = self.fake_gerrit.addFakeChange('org/nonvoting-project',

  1667.                                            'master', 'A')

  1668.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  1669.         self.executor_server.failJob('nonvoting-project-test2', A)

  1670. 

  1671.         self.waitUntilSettled()

  1672. 

  1673.         self.assertEqual(A.data['status'], 'NEW')

  1674.         self.assertEqual(A.reported, 1)

  1675. 

  1676.         self.assertEqual(

  1677.             self.getJobFromHistory('nonvoting-project-test1').result,

  1678.             'SUCCESS')

  1679.         self.assertEqual(

  1680.             self.getJobFromHistory('nonvoting-project-test2').result,

  1681.             'FAILURE')

  1682. 

  1683.         self.assertFalse(self.getJobFromHistory('nonvoting-project-test1').

  1684.                          parameters['zuul']['voting'])

  1685.         self.assertFalse(self.getJobFromHistory('nonvoting-project-test2').

  1686.                          parameters['zuul']['voting'])

  1687.         self.assertEqual(A.patchsets[0]['approvals'][0]['value'], "1")

  1688. 

  1689.     @simple_layout('layouts/nonvoting-job.yaml')

  1690.     def test_nonvoting_job(self):

  1691.         "Test that non-voting jobs don't vote."

  1692. 

  1693.         A = self.fake_gerrit.addFakeChange('org/nonvoting-project',

  1694.                                            'master', 'A')

  1695.         A.addApproval('Code-Review', 2)

  1696.         self.executor_server.failJob('nonvoting-project-test2', A)

  1697.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  1698. 

  1699.         self.waitUntilSettled()

  1700. 

  1701.         self.assertEqual(A.data['status'], 'MERGED')

  1702.         self.assertEqual(A.reported, 2)

  1703.         self.assertEqual(

  1704.             self.getJobFromHistory('nonvoting-project-merge').result,

  1705.             'SUCCESS')

  1706.         self.assertEqual(

  1707.             self.getJobFromHistory('nonvoting-project-test1').result,

  1708.             'SUCCESS')

  1709.         self.assertEqual(

  1710.             self.getJobFromHistory('nonvoting-project-test2').result,

  1711.             'FAILURE')

  1712. 

  1713.         self.assertTrue(self.getJobFromHistory('nonvoting-project-merge').

  1714.                         parameters['zuul']['voting'])

  1715.         self.assertTrue(self.getJobFromHistory('nonvoting-project-test1').

  1716.                         parameters['zuul']['voting'])

  1717.         self.assertFalse(self.getJobFromHistory('nonvoting-project-test2').

  1718.                          parameters['zuul']['voting'])

  1719. 

  1720.     def test_check_queue_success(self):

  1721.         "Test successful check queue jobs."

  1722. 

  1723.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1724.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  1725. 

  1726.         self.waitUntilSettled()

  1727. 

  1728.         self.assertEqual(A.data['status'], 'NEW')

  1729.         self.assertEqual(A.reported, 1)

  1730.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  1731.                          'SUCCESS')

  1732.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  1733.                          'SUCCESS')

  1734.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  1735.                          'SUCCESS')

  1736. 

  1737.     def test_check_queue_failure(self):

  1738.         "Test failed check queue jobs."

  1739. 

  1740.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1741.         self.executor_server.failJob('project-test2', A)

  1742.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  1743. 

  1744.         self.waitUntilSettled()

  1745. 

  1746.         self.assertEqual(A.data['status'], 'NEW')

  1747.         self.assertEqual(A.reported, 1)

  1748.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  1749.                          'SUCCESS')

  1750.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  1751.                          'SUCCESS')

  1752.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  1753.                          'FAILURE')

  1754. 

  1755.     @simple_layout('layouts/autohold.yaml')

  1756.     def test_autohold(self):

  1757.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  1758.                                           self.gearman_server.port)

  1759.         self.addCleanup(client.shutdown)

  1760.         r = client.autohold('tenant-one', 'org/project', 'project-test2',

  1761.                             "", "", "reason text", 1)

  1762.         self.assertTrue(r)

  1763. 

  1764.         # There should be a record in ZooKeeper

  1765.         request_list = self.scheds.first.sched.zk.getHoldRequests()

  1766.         self.assertEqual(1, len(request_list))

  1767.         request = self.scheds.first.sched.zk.getHoldRequest(request_list[0])

  1768.         self.assertIsNotNone(request)

  1769.         self.assertEqual('tenant-one', request.tenant)

  1770.         self.assertEqual('review.example.com/org/project', request.project)

  1771.         self.assertEqual('project-test2', request.job)

  1772.         self.assertEqual('reason text', request.reason)

  1773.         self.assertEqual(1, request.max_count)

  1774.         self.assertEqual(0, request.current_count)

  1775.         self.assertEqual([], request.nodes)

  1776. 

  1777.         # First check that successful jobs do not autohold

  1778.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1779.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  1780. 

  1781.         self.waitUntilSettled()

  1782. 

  1783.         self.assertEqual(A.data['status'], 'NEW')

  1784.         self.assertEqual(A.reported, 1)

  1785.         # project-test2

  1786.         self.assertEqual(self.history[0].result, 'SUCCESS')

  1787. 

  1788.         # Check nodepool for a held node

  1789.         held_node = None

  1790.         for node in self.fake_nodepool.getNodes():

  1791.             if node['state'] == zuul.model.STATE_HOLD:

  1792.                 held_node = node

  1793.                 break

  1794.         self.assertIsNone(held_node)

  1795. 

  1796.         # Now test that failed jobs are autoheld

  1797.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  1798.         self.executor_server.failJob('project-test2', B)

  1799.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  1800. 

  1801.         self.waitUntilSettled()

  1802. 

  1803.         self.assertEqual(B.data['status'], 'NEW')

  1804.         self.assertEqual(B.reported, 1)

  1805.         # project-test2

  1806.         self.assertEqual(self.history[1].result, 'FAILURE')

  1807. 

  1808.         # Check nodepool for a held node

  1809.         held_node = None

  1810.         for node in self.fake_nodepool.getNodes():

  1811.             if node['state'] == zuul.model.STATE_HOLD:

  1812.                 held_node = node

  1813.                 break

  1814.         self.assertIsNotNone(held_node)

  1815.         # Validate node has recorded the failed job

  1816.         self.assertEqual(

  1817.             held_node['hold_job'],

  1818.             " ".join(['tenant-one',

  1819.                       'review.example.com/org/project',

  1820.                       'project-test2', '.*'])

  1821.         )

  1822.         self.assertEqual(held_node['comment'], "reason text")

  1823. 

  1824.         # The hold request current_count should have incremented

  1825.         # and we should have recorded the held node ID.

  1826.         request2 = self.scheds.first.sched.zk.getHoldRequest(request.id)

  1827.         self.assertEqual(request.current_count + 1, request2.current_count)

  1828.         self.assertEqual(1, len(request2.nodes))

  1829.         self.assertEqual(1, len(request2.nodes[0]["nodes"]))

  1830. 

  1831.         # Another failed change should not hold any more nodes

  1832.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  1833.         self.executor_server.failJob('project-test2', C)

  1834.         self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

  1835.         self.waitUntilSettled()

  1836.         self.assertEqual(C.data['status'], 'NEW')

  1837.         self.assertEqual(C.reported, 1)

  1838.         # project-test2

  1839.         self.assertEqual(self.history[2].result, 'FAILURE')

  1840. 

  1841.         held_nodes = 0

  1842.         for node in self.fake_nodepool.getNodes():

  1843.             if node['state'] == zuul.model.STATE_HOLD:

  1844.                 held_nodes += 1

  1845.         self.assertEqual(held_nodes, 1)

  1846. 

  1847.         # request current_count should not have changed

  1848.         request3 = self.scheds.first.sched.zk.getHoldRequest(request2.id)

  1849.         self.assertEqual(request2.current_count, request3.current_count)

  1850. 

  1851.         # Deleting hold request should set held nodes to used

  1852.         self.scheds.first.sched.zk.deleteHoldRequest(request3)

  1853.         node_states = [n['state'] for n in self.fake_nodepool.getNodes()]

  1854.         self.assertEqual(3, len(node_states))

  1855.         self.assertEqual([zuul.model.STATE_USED] * 3, node_states)

  1856. 

  1857.     @simple_layout('layouts/autohold.yaml')

  1858.     def test_autohold_info(self):

  1859.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  1860.                                           self.gearman_server.port)

  1861.         self.addCleanup(client.shutdown)

  1862. 

  1863.         # Empty dict should be returned for "not found"

  1864.         request = client.autohold_info("XxXxX")

  1865.         self.assertEqual({}, request)

  1866. 

  1867.         r = client.autohold('tenant-one', 'org/project', 'project-test2',

  1868.                             "", "", "reason text", 1)

  1869.         self.assertTrue(r)

  1870. 

  1871.         # There should be a record in ZooKeeper

  1872.         request_list = self.scheds.first.sched.zk.getHoldRequests()

  1873.         self.assertEqual(1, len(request_list))

  1874.         request = self.scheds.first.sched.zk.getHoldRequest(request_list[0])

  1875.         self.assertIsNotNone(request)

  1876. 

  1877.         request = client.autohold_info(request.id)

  1878.         self.assertNotEqual({}, request)

  1879.         self.assertEqual('tenant-one', request['tenant'])

  1880.         self.assertEqual('review.example.com/org/project', request['project'])

  1881.         self.assertEqual('project-test2', request['job'])

  1882.         self.assertEqual('reason text', request['reason'])

  1883.         self.assertEqual(1, request['max_count'])

  1884.         self.assertEqual(0, request['current_count'])

  1885. 

  1886.     @simple_layout('layouts/autohold.yaml')

  1887.     def test_autohold_delete(self):

  1888.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  1889.                                           self.gearman_server.port)

  1890.         self.addCleanup(client.shutdown)

  1891.         r = client.autohold('tenant-one', 'org/project', 'project-test2',

  1892.                             "", "", "reason text", 1)

  1893.         self.assertTrue(r)

  1894. 

  1895.         # There should be a record in ZooKeeper

  1896.         request_list = self.scheds.first.sched.zk.getHoldRequests()

  1897.         self.assertEqual(1, len(request_list))

  1898.         request = self.scheds.first.sched.zk.getHoldRequest(request_list[0])

  1899.         self.assertIsNotNone(request)

  1900. 

  1901.         # Delete and verify no more requests

  1902.         self.assertTrue(client.autohold_delete(request.id))

  1903.         request_list = self.scheds.first.sched.zk.getHoldRequests()

  1904.         self.assertEqual([], request_list)

  1905. 

  1906.     def _test_autohold_scoped(self, change_obj, change, ref):

  1907.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  1908.                                           self.gearman_server.port)

  1909.         self.addCleanup(client.shutdown)

  1910. 

  1911.         # create two changes on the same project, and autohold request

  1912.         # for one of them.

  1913.         other = self.fake_gerrit.addFakeChange(

  1914.             'org/project', 'master', 'other'

  1915.         )

  1916. 

  1917.         r = client.autohold('tenant-one', 'org/project', 'project-test2',

  1918.                             str(change), ref, "reason text", 1)

  1919.         self.assertTrue(r)

  1920. 

  1921.         # First, check that an unrelated job does not trigger autohold, even

  1922.         # when it failed

  1923.         self.executor_server.failJob('project-test2', other)

  1924.         self.fake_gerrit.addEvent(other.getPatchsetCreatedEvent(1))

  1925. 

  1926.         self.waitUntilSettled()

  1927. 

  1928.         self.assertEqual(other.data['status'], 'NEW')

  1929.         self.assertEqual(other.reported, 1)

  1930.         # project-test2

  1931.         self.assertEqual(self.history[0].result, 'FAILURE')

  1932. 

  1933.         # Check nodepool for a held node

  1934.         held_node = None

  1935.         for node in self.fake_nodepool.getNodes():

  1936.             if node['state'] == zuul.model.STATE_HOLD:

  1937.                 held_node = node

  1938.                 break

  1939.         self.assertIsNone(held_node)

  1940. 

  1941.         # And then verify that failed job for the defined change

  1942.         # triggers the autohold

  1943. 

  1944.         self.executor_server.failJob('project-test2', change_obj)

  1945.         self.fake_gerrit.addEvent(change_obj.getPatchsetCreatedEvent(1))

  1946. 

  1947.         self.waitUntilSettled()

  1948. 

  1949.         self.assertEqual(change_obj.data['status'], 'NEW')

  1950.         self.assertEqual(change_obj.reported, 1)

  1951.         # project-test2

  1952.         self.assertEqual(self.history[1].result, 'FAILURE')

  1953. 

  1954.         # Check nodepool for a held node

  1955.         held_node = None

  1956.         for node in self.fake_nodepool.getNodes():

  1957.             if node['state'] == zuul.model.STATE_HOLD:

  1958.                 held_node = node

  1959.                 break

  1960.         self.assertIsNotNone(held_node)

  1961. 

  1962.         # Validate node has recorded the failed job

  1963.         if change != "":

  1964.             ref = "refs/changes/%s/%s/.*" % (

  1965.                 str(change_obj.number)[-1:], str(change_obj.number)

  1966.             )

  1967. 

  1968.         self.assertEqual(

  1969.             held_node['hold_job'],

  1970.             " ".join(['tenant-one',

  1971.                       'review.example.com/org/project',

  1972.                       'project-test2', ref])

  1973.         )

  1974.         self.assertEqual(held_node['comment'], "reason text")

  1975. 

  1976.     @simple_layout('layouts/autohold.yaml')

  1977.     def test_autohold_change(self):

  1978.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1979. 

  1980.         self._test_autohold_scoped(A, change=A.number, ref="")

  1981. 

  1982.     @simple_layout('layouts/autohold.yaml')

  1983.     def test_autohold_ref(self):

  1984.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1985.         ref = A.data['currentPatchSet']['ref']

  1986.         self._test_autohold_scoped(A, change="", ref=ref)

  1987. 

  1988.     @simple_layout('layouts/autohold.yaml')

  1989.     def test_autohold_scoping(self):

  1990.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  1991.                                           self.gearman_server.port)

  1992.         self.addCleanup(client.shutdown)

  1993. 

  1994.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  1995. 

  1996.         # create three autohold requests, scoped to job, change and

  1997.         # a specific ref

  1998.         change = str(A.number)

  1999.         ref = A.data['currentPatchSet']['ref']

  2000.         r1 = client.autohold('tenant-one', 'org/project', 'project-test2',

  2001.                              "", "", "reason text", 1)

  2002.         self.assertTrue(r1)

  2003.         r2 = client.autohold('tenant-one', 'org/project', 'project-test2',

  2004.                              change, "", "reason text", 1)

  2005.         self.assertTrue(r2)

  2006.         r3 = client.autohold('tenant-one', 'org/project', 'project-test2',

  2007.                              "", ref, "reason text", 1)

  2008.         self.assertTrue(r3)

  2009. 

  2010.         # Fail 3 jobs for the same change, and verify that the autohold

  2011.         # requests are fullfilled in the expected order: from the most

  2012.         # specific towards the most generic one.

  2013. 

  2014.         def _fail_job_and_verify_autohold_request(change_obj, ref_filter):

  2015.             self.executor_server.failJob('project-test2', change_obj)

  2016.             self.fake_gerrit.addEvent(change_obj.getPatchsetCreatedEvent(1))

  2017. 

  2018.             self.waitUntilSettled()

  2019. 

  2020.             # Check nodepool for a held node

  2021.             held_node = None

  2022.             for node in self.fake_nodepool.getNodes():

  2023.                 if node['state'] == zuul.model.STATE_HOLD:

  2024.                     held_node = node

  2025.                     break

  2026.             self.assertIsNotNone(held_node)

  2027. 

  2028.             self.assertEqual(

  2029.                 held_node['hold_job'],

  2030.                 " ".join(['tenant-one',

  2031.                           'review.example.com/org/project',

  2032.                           'project-test2', ref_filter])

  2033.             )

  2034.             self.assertFalse(held_node['_lock'], "Node %s is locked" %

  2035.                              (node['_oid'],))

  2036.             self.fake_nodepool.removeNode(held_node)

  2037. 

  2038.         _fail_job_and_verify_autohold_request(A, ref)

  2039. 

  2040.         ref = "refs/changes/%s/%s/.*" % (str(change)[-1:], str(change))

  2041.         _fail_job_and_verify_autohold_request(A, ref)

  2042.         _fail_job_and_verify_autohold_request(A, ".*")

  2043. 

  2044.     @simple_layout('layouts/autohold.yaml')

  2045.     def test_autohold_ignores_aborted_jobs(self):

  2046.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  2047.                                           self.gearman_server.port)

  2048.         self.addCleanup(client.shutdown)

  2049.         r = client.autohold('tenant-one', 'org/project', 'project-test2',

  2050.                             "", "", "reason text", 1)

  2051.         self.assertTrue(r)

  2052. 

  2053.         self.executor_server.hold_jobs_in_build = True

  2054. 

  2055.         # Create a change that will have its job aborted

  2056.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2057.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  2058.         self.waitUntilSettled()

  2059. 

  2060.         # Creating new patchset on change A will abort A,1's job because

  2061.         # a new patchset arrived replacing A,1 with A,2.

  2062.         A.addPatchset()

  2063.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))

  2064. 

  2065.         self.waitUntilSettled()

  2066.         self.executor_server.hold_jobs_in_build = False

  2067.         self.executor_server.release()

  2068.         self.waitUntilSettled()

  2069. 

  2070.         self.assertEqual(A.data['status'], 'NEW')

  2071.         # Note only the successful job for A,2 will report as we don't

  2072.         # report aborted builds for old patchsets.

  2073.         self.assertEqual(A.reported, 1)

  2074.         # A,1 project-test2

  2075.         self.assertEqual(self.history[0].result, 'ABORTED')

  2076.         # A,2 project-test2

  2077.         self.assertEqual(self.history[1].result, 'SUCCESS')

  2078. 

  2079.         # Check nodepool for a held node

  2080.         held_node = None

  2081.         for node in self.fake_nodepool.getNodes():

  2082.             if node['state'] == zuul.model.STATE_HOLD:

  2083.                 held_node = node

  2084.                 break

  2085.         self.assertIsNone(held_node)

  2086. 

  2087.     @simple_layout('layouts/autohold.yaml')

  2088.     def test_autohold_hold_expiration(self):

  2089.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  2090.                                           self.gearman_server.port)

  2091.         self.addCleanup(client.shutdown)

  2092.         r = client.autohold('tenant-one', 'org/project', 'project-test2',

  2093.                             "", "", "reason text", 1, node_hold_expiration=30)

  2094.         self.assertTrue(r)

  2095. 

  2096.         # Hold a failed job

  2097.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  2098.         self.executor_server.failJob('project-test2', B)

  2099.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  2100. 

  2101.         self.waitUntilSettled()

  2102. 

  2103.         self.assertEqual(B.data['status'], 'NEW')

  2104.         self.assertEqual(B.reported, 1)

  2105.         # project-test2

  2106.         self.assertEqual(self.history[0].result, 'FAILURE')

  2107. 

  2108.         # Check nodepool for a held node

  2109.         held_node = None

  2110.         for node in self.fake_nodepool.getNodes():

  2111.             if node['state'] == zuul.model.STATE_HOLD:

  2112.                 held_node = node

  2113.                 break

  2114.         self.assertIsNotNone(held_node)

  2115. 

  2116.         # Validate node has hold_expiration property

  2117.         self.assertEqual(int(held_node['hold_expiration']), 30)

  2118. 

  2119.     @simple_layout('layouts/autohold.yaml')

  2120.     def test_autohold_list(self):

  2121.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  2122.                                           self.gearman_server.port)

  2123.         self.addCleanup(client.shutdown)

  2124. 

  2125.         r = client.autohold('tenant-one', 'org/project', 'project-test2',

  2126.                             "", "", "reason text", 1)

  2127.         self.assertTrue(r)

  2128. 

  2129.         autohold_requests = client.autohold_list()

  2130.         self.assertNotEqual([], autohold_requests)

  2131.         self.assertEqual(1, len(autohold_requests))

  2132. 

  2133.         request = autohold_requests[0]

  2134.         self.assertEqual('tenant-one', request['tenant'])

  2135.         self.assertIn('org/project', request['project'])

  2136.         self.assertEqual('project-test2', request['job'])

  2137.         self.assertEqual(".*", request['ref_filter'])

  2138.         self.assertEqual("reason text", request['reason'])

  2139. 

  2140.     @simple_layout('layouts/autohold.yaml')

  2141.     def test_autohold_request_expiration(self):

  2142.         orig_exp = Scheduler.EXPIRED_HOLD_REQUEST_TTL

  2143. 

  2144.         def reset_exp():

  2145.             self.scheds.first.sched.EXPIRED_HOLD_REQUEST_TTL = orig_exp

  2146. 

  2147.         self.addCleanup(reset_exp)

  2148. 

  2149.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  2150.                                           self.gearman_server.port)

  2151.         self.addCleanup(client.shutdown)

  2152. 

  2153.         # Temporarily shorten the hold request expiration time

  2154.         r = client.autohold('tenant-one', 'org/project', 'project-test2',

  2155.                             "", "", "reason text", 1, 1)

  2156.         self.assertTrue(r)

  2157. 

  2158.         autohold_requests = client.autohold_list()

  2159.         self.assertEqual(1, len(autohold_requests))

  2160.         req = autohold_requests[0]

  2161.         self.assertIsNone(req['expired'])

  2162. 

  2163.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2164.         self.executor_server.failJob('project-test2', A)

  2165.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  2166.         self.waitUntilSettled()

  2167. 

  2168.         autohold_requests = client.autohold_list()

  2169.         self.assertEqual(1, len(autohold_requests))

  2170.         req = autohold_requests[0]

  2171.         self.assertIsNotNone(req['expired'])

  2172. 

  2173.         # Temporarily shorten hold time so that the hold request can be

  2174.         # auto-deleted (which is done on another test failure). And wait

  2175.         # long enough for nodes to expire and request to delete.

  2176.         self.scheds.first.sched.EXPIRED_HOLD_REQUEST_TTL = 1

  2177.         time.sleep(3)

  2178. 

  2179.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  2180.         self.executor_server.failJob('project-test2', B)

  2181.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  2182.         self.waitUntilSettled()

  2183. 

  2184.         for _ in iterate_timeout(10, 'hold request expiration'):

  2185.             if len(client.autohold_list()) == 0:

  2186.                 break

  2187. 

  2188.     @simple_layout('layouts/three-projects.yaml')

  2189.     def test_dependent_behind_dequeue(self):

  2190.         # This particular test does a large amount of merges and needs a little

  2191.         # more time to complete

  2192.         self.wait_timeout = 120

  2193.         "test that dependent changes behind dequeued changes work"

  2194.         # This complicated test is a reproduction of a real life bug

  2195.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  2196. 

  2197.         self.executor_server.hold_jobs_in_build = True

  2198.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  2199.         B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')

  2200.         C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')

  2201.         D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')

  2202.         E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E')

  2203.         F = self.fake_gerrit.addFakeChange('org/project3', 'master', 'F')

  2204.         D.setDependsOn(C, 1)

  2205.         E.setDependsOn(D, 1)

  2206.         A.addApproval('Code-Review', 2)

  2207.         B.addApproval('Code-Review', 2)

  2208.         C.addApproval('Code-Review', 2)

  2209.         D.addApproval('Code-Review', 2)

  2210.         E.addApproval('Code-Review', 2)

  2211.         F.addApproval('Code-Review', 2)

  2212. 

  2213.         A.fail_merge = True

  2214. 

  2215.         # Change object re-use in the gerrit trigger is hidden if

  2216.         # changes are added in quick succession; waiting makes it more

  2217.         # like real life.

  2218.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2219.         self.waitUntilSettled()

  2220.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  2221.         self.waitUntilSettled()

  2222. 

  2223.         self.executor_server.release('.*-merge')

  2224.         self.waitUntilSettled()

  2225.         self.executor_server.release('.*-merge')

  2226.         self.waitUntilSettled()

  2227. 

  2228.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  2229.         self.waitUntilSettled()

  2230.         self.fake_gerrit.addEvent(D.addApproval('Approved', 1))

  2231.         self.waitUntilSettled()

  2232.         self.fake_gerrit.addEvent(E.addApproval('Approved', 1))

  2233.         self.waitUntilSettled()

  2234.         self.fake_gerrit.addEvent(F.addApproval('Approved', 1))

  2235.         self.waitUntilSettled()

  2236. 

  2237.         self.executor_server.release('.*-merge')

  2238.         self.waitUntilSettled()

  2239.         self.executor_server.release('.*-merge')

  2240.         self.waitUntilSettled()

  2241.         self.executor_server.release('.*-merge')

  2242.         self.waitUntilSettled()

  2243.         self.executor_server.release('.*-merge')

  2244.         self.waitUntilSettled()

  2245. 

  2246.         # all jobs running

  2247. 

  2248.         # Grab pointers to the jobs we want to release before

  2249.         # releasing any, because list indexes may change as

  2250.         # the jobs complete.

  2251.         a, b, c = self.builds[:3]

  2252.         a.release()

  2253.         b.release()

  2254.         c.release()

  2255.         self.waitUntilSettled()

  2256. 

  2257.         self.executor_server.hold_jobs_in_build = False

  2258.         self.executor_server.release()

  2259.         self.waitUntilSettled()

  2260. 

  2261.         self.assertEqual(A.data['status'], 'NEW')

  2262.         self.assertEqual(B.data['status'], 'MERGED')

  2263.         self.assertEqual(C.data['status'], 'MERGED')

  2264.         self.assertEqual(D.data['status'], 'MERGED')

  2265.         self.assertEqual(E.data['status'], 'MERGED')

  2266.         self.assertEqual(F.data['status'], 'MERGED')

  2267. 

  2268.         self.assertEqual(A.reported, 2)

  2269.         self.assertEqual(B.reported, 2)

  2270.         self.assertEqual(C.reported, 2)

  2271.         self.assertEqual(D.reported, 2)

  2272.         self.assertEqual(E.reported, 2)

  2273.         self.assertEqual(F.reported, 2)

  2274. 

  2275.         self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 15)

  2276.         self.assertEqual(len(self.history), 44)

  2277. 

  2278.     def test_merger_repack(self):

  2279.         "Test that the merger works after a repack"

  2280. 

  2281.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2282.         A.addApproval('Code-Review', 2)

  2283.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2284.         self.waitUntilSettled()

  2285.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  2286.                          'SUCCESS')

  2287.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  2288.                          'SUCCESS')

  2289.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  2290.                          'SUCCESS')

  2291.         self.assertEqual(A.data['status'], 'MERGED')

  2292.         self.assertEqual(A.reported, 2)

  2293.         self.assertEmptyQueues()

  2294.         self.build_history = []

  2295. 

  2296.         path = os.path.join(self.merger_src_root, "review.example.com",

  2297.                             "org/project")

  2298.         if os.path.exists(path):

  2299.             repack_repo(path)

  2300.         path = os.path.join(self.executor_src_root, "review.example.com",

  2301.                             "org/project")

  2302.         if os.path.exists(path):

  2303.             repack_repo(path)

  2304. 

  2305.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2306.         A.addApproval('Code-Review', 2)

  2307.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2308.         self.waitUntilSettled()

  2309.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  2310.                          'SUCCESS')

  2311.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  2312.                          'SUCCESS')

  2313.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  2314.                          'SUCCESS')

  2315.         self.assertEqual(A.data['status'], 'MERGED')

  2316.         self.assertEqual(A.reported, 2)

  2317. 

  2318.     def test_merger_repack_large_change(self):

  2319.         "Test that the merger works with large changes after a repack"

  2320.         # https://bugs.executepad.net/zuul/+bug/1078946

  2321.         # This test assumes the repo is already cloned; make sure it is

  2322.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  2323.         trusted, project = tenant.getProject('org/project')

  2324.         url = self.fake_gerrit.getGitUrl(project)

  2325.         self.executor_server.merger._addProject('review.example.com',

  2326.                                                 'org/project', url, None, None)

  2327.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2328.         A.addPatchset(large=True)

  2329.         # TODOv3(jeblair): add hostname to upstream root

  2330.         path = os.path.join(self.upstream_root, 'org/project')

  2331.         repack_repo(path)

  2332.         path = os.path.join(self.merger_src_root, 'review.example.com',

  2333.                             'org/project')

  2334.         if os.path.exists(path):

  2335.             repack_repo(path)

  2336.         path = os.path.join(self.executor_src_root, 'review.example.com',

  2337.                             'org/project')

  2338.         if os.path.exists(path):

  2339.             repack_repo(path)

  2340. 

  2341.         A.addApproval('Code-Review', 2)

  2342.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2343.         self.waitUntilSettled()

  2344.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  2345.                          'SUCCESS')

  2346.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  2347.                          'SUCCESS')

  2348.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  2349.                          'SUCCESS')

  2350.         self.assertEqual(A.data['status'], 'MERGED')

  2351.         self.assertEqual(A.reported, 2)

  2352. 

  2353.     def test_new_patchset_dequeues_old(self):

  2354.         "Test that a new patchset causes the old to be dequeued"

  2355.         # D -> C (depends on B) -> B (depends on A) -> A -> M

  2356.         self.executor_server.hold_jobs_in_build = True

  2357.         M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')

  2358.         M.setMerged()

  2359. 

  2360.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2361.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  2362.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  2363.         D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')

  2364.         A.addApproval('Code-Review', 2)

  2365.         B.addApproval('Code-Review', 2)

  2366.         C.addApproval('Code-Review', 2)

  2367.         D.addApproval('Code-Review', 2)

  2368. 

  2369.         C.setDependsOn(B, 1)

  2370.         B.setDependsOn(A, 1)

  2371.         A.setDependsOn(M, 1)

  2372. 

  2373.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2374.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  2375.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  2376.         self.fake_gerrit.addEvent(D.addApproval('Approved', 1))

  2377.         self.waitUntilSettled()

  2378. 

  2379.         B.addPatchset()

  2380.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))

  2381.         self.waitUntilSettled()

  2382. 

  2383.         self.executor_server.hold_jobs_in_build = False

  2384.         self.executor_server.release()

  2385.         self.waitUntilSettled()

  2386. 

  2387.         self.assertEqual(A.data['status'], 'MERGED')

  2388.         self.assertEqual(A.reported, 2)

  2389.         self.assertEqual(B.data['status'], 'NEW')

  2390.         self.assertEqual(B.reported, 2)

  2391.         self.assertEqual(C.data['status'], 'NEW')

  2392.         self.assertEqual(C.reported, 2)

  2393.         self.assertEqual(D.data['status'], 'MERGED')

  2394.         self.assertEqual(D.reported, 2)

  2395.         self.assertEqual(len(self.history), 9)  # 3 each for A, B, D.

  2396. 

  2397.     def test_new_patchset_check(self):

  2398.         "Test a new patchset in check"

  2399. 

  2400.         self.executor_server.hold_jobs_in_build = True

  2401. 

  2402.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2403.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  2404.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  2405.         check_pipeline = tenant.layout.pipelines['check']

  2406. 

  2407.         # Add two git-dependent changes

  2408.         B.setDependsOn(A, 1)

  2409.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  2410.         self.waitUntilSettled()

  2411.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  2412.         self.waitUntilSettled()

  2413. 

  2414.         # A live item, and a non-live/live pair

  2415.         items = check_pipeline.getAllItems()

  2416.         self.assertEqual(len(items), 3)

  2417. 

  2418.         self.assertEqual(items[0].change.number, '1')

  2419.         self.assertEqual(items[0].change.patchset, '1')

  2420.         self.assertFalse(items[0].live)

  2421. 

  2422.         self.assertEqual(items[1].change.number, '2')

  2423.         self.assertEqual(items[1].change.patchset, '1')

  2424.         self.assertTrue(items[1].live)

  2425. 

  2426.         self.assertEqual(items[2].change.number, '1')

  2427.         self.assertEqual(items[2].change.patchset, '1')

  2428.         self.assertTrue(items[2].live)

  2429. 

  2430.         # Add a new patchset to A

  2431.         A.addPatchset()

  2432.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))

  2433.         self.waitUntilSettled()

  2434. 

  2435.         # The live copy of A,1 should be gone, but the non-live and B

  2436.         # should continue, and we should have a new A,2

  2437.         items = check_pipeline.getAllItems()

  2438.         self.assertEqual(len(items), 3)

  2439. 

  2440.         self.assertEqual(items[0].change.number, '1')

  2441.         self.assertEqual(items[0].change.patchset, '1')

  2442.         self.assertFalse(items[0].live)

  2443. 

  2444.         self.assertEqual(items[1].change.number, '2')

  2445.         self.assertEqual(items[1].change.patchset, '1')

  2446.         self.assertTrue(items[1].live)

  2447. 

  2448.         self.assertEqual(items[2].change.number, '1')

  2449.         self.assertEqual(items[2].change.patchset, '2')

  2450.         self.assertTrue(items[2].live)

  2451. 

  2452.         # Add a new patchset to B

  2453.         B.addPatchset()

  2454.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))

  2455.         self.waitUntilSettled()

  2456. 

  2457.         # The live copy of B,1 should be gone, and it's non-live copy of A,1

  2458.         # but we should have a new B,2 (still based on A,1)

  2459.         items = check_pipeline.getAllItems()

  2460.         self.assertEqual(len(items), 3)

  2461. 

  2462.         self.assertEqual(items[0].change.number, '1')

  2463.         self.assertEqual(items[0].change.patchset, '2')

  2464.         self.assertTrue(items[0].live)

  2465. 

  2466.         self.assertEqual(items[1].change.number, '1')

  2467.         self.assertEqual(items[1].change.patchset, '1')

  2468.         self.assertFalse(items[1].live)

  2469. 

  2470.         self.assertEqual(items[2].change.number, '2')

  2471.         self.assertEqual(items[2].change.patchset, '2')

  2472.         self.assertTrue(items[2].live)

  2473. 

  2474.         self.builds[0].release()

  2475.         self.waitUntilSettled()

  2476.         self.builds[0].release()

  2477.         self.waitUntilSettled()

  2478.         self.executor_server.hold_jobs_in_build = False

  2479.         self.executor_server.release()

  2480.         self.waitUntilSettled()

  2481. 

  2482.         self.assertEqual(A.reported, 1)

  2483.         self.assertEqual(B.reported, 1)

  2484.         self.assertEqual(self.history[0].result, 'ABORTED')

  2485.         self.assertEqual(self.history[0].changes, '1,1')

  2486.         self.assertEqual(self.history[1].result, 'ABORTED')

  2487.         self.assertEqual(self.history[1].changes, '1,1 2,1')

  2488.         self.assertEqual(self.history[2].result, 'SUCCESS')

  2489.         self.assertEqual(self.history[2].changes, '1,2')

  2490.         self.assertEqual(self.history[3].result, 'SUCCESS')

  2491.         self.assertEqual(self.history[3].changes, '1,1 2,2')

  2492. 

  2493.     def test_abandoned_gate(self):

  2494.         "Test that an abandoned change is dequeued from gate"

  2495. 

  2496.         self.executor_server.hold_jobs_in_build = True

  2497. 

  2498.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2499.         A.addApproval('Code-Review', 2)

  2500.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2501.         self.waitUntilSettled()

  2502.         self.assertEqual(len(self.builds), 1, "One job being built (on hold)")

  2503.         self.assertEqual(self.builds[0].name, 'project-merge')

  2504. 

  2505.         self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())

  2506.         self.waitUntilSettled()

  2507. 

  2508.         self.executor_server.release('.*-merge')

  2509.         self.waitUntilSettled()

  2510. 

  2511.         self.assertBuilds([])

  2512.         self.assertHistory([

  2513.             dict(name='project-merge', result='ABORTED', changes='1,1')],

  2514.             ordered=False)

  2515.         self.assertEqual(A.reported, 1,

  2516.                          "Abandoned gate change should report only start")

  2517. 

  2518.     def test_abandoned_check(self):

  2519.         "Test that an abandoned change is dequeued from check"

  2520. 

  2521.         self.executor_server.hold_jobs_in_build = True

  2522. 

  2523.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2524.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  2525.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  2526.         check_pipeline = tenant.layout.pipelines['check']

  2527. 

  2528.         # Add two git-dependent changes

  2529.         B.setDependsOn(A, 1)

  2530.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  2531.         self.waitUntilSettled()

  2532.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  2533.         self.waitUntilSettled()

  2534.         # A live item, and a non-live/live pair

  2535.         items = check_pipeline.getAllItems()

  2536.         self.assertEqual(len(items), 3)

  2537. 

  2538.         self.assertEqual(items[0].change.number, '1')

  2539.         self.assertFalse(items[0].live)

  2540. 

  2541.         self.assertEqual(items[1].change.number, '2')

  2542.         self.assertTrue(items[1].live)

  2543. 

  2544.         self.assertEqual(items[2].change.number, '1')

  2545.         self.assertTrue(items[2].live)

  2546. 

  2547.         # Abandon A

  2548.         self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())

  2549.         self.waitUntilSettled()

  2550. 

  2551.         # The live copy of A should be gone, but the non-live and B

  2552.         # should continue

  2553.         items = check_pipeline.getAllItems()

  2554.         self.assertEqual(len(items), 2)

  2555. 

  2556.         self.assertEqual(items[0].change.number, '1')

  2557.         self.assertFalse(items[0].live)

  2558. 

  2559.         self.assertEqual(items[1].change.number, '2')

  2560.         self.assertTrue(items[1].live)

  2561. 

  2562.         self.executor_server.hold_jobs_in_build = False

  2563.         self.executor_server.release()

  2564.         self.waitUntilSettled()

  2565. 

  2566.         self.assertEqual(len(self.history), 4)

  2567.         self.assertEqual(self.history[0].result, 'ABORTED',

  2568.                          'Build should have been aborted')

  2569.         self.assertEqual(A.reported, 0, "Abandoned change should not report")

  2570.         self.assertEqual(B.reported, 1, "Change should report")

  2571. 

  2572.     def test_cancel_starting_build(self):

  2573.         "Test that a canceled build that is not processed yet is removed"

  2574. 

  2575.         self.executor_server.hold_jobs_in_start = True

  2576.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2577. 

  2578.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  2579.         for _ in iterate_timeout(30, 'Wait for build to be in starting phase'):

  2580.             if self.executor_server.job_workers:

  2581.                 break

  2582. 

  2583.         # Abandon change to cancel build

  2584.         self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())

  2585. 

  2586.         for _ in iterate_timeout(30, 'Wait for executor:stop request'):

  2587.             stop_jobs = [x for x in self.gearman_server.jobs_history

  2588.                          if b'executor:stop' in x.name]

  2589.             if stop_jobs:

  2590.                 break

  2591. 

  2592.         self.executor_server.hold_jobs_in_start = False

  2593.         self.waitUntilSettled()

  2594. 

  2595.         self.assertHistory([

  2596.             dict(name='project-merge', result='ABORTED')

  2597.         ])

  2598. 

  2599.     def test_abandoned_not_timer(self):

  2600.         "Test that an abandoned change does not cancel timer jobs"

  2601.         # This test can not use simple_layout because it must start

  2602.         # with a configuration which does not include a

  2603.         # timer-triggered job so that we have an opportunity to set

  2604.         # the hold flag before the first job.

  2605.         self.executor_server.hold_jobs_in_build = True

  2606.         # Start timer trigger - also org/project

  2607.         self.commitConfigUpdate('common-config', 'layouts/idle.yaml')

  2608.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  2609.         # The pipeline triggers every second, so we should have seen

  2610.         # several by now.

  2611.         time.sleep(5)

  2612.         self.waitUntilSettled()

  2613.         # Stop queuing timer triggered jobs so that the assertions

  2614.         # below don't race against more jobs being queued.

  2615.         # Must be in same repo, so overwrite config with another one

  2616.         self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')

  2617.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  2618.         self.waitUntilSettled()

  2619.         # If APScheduler is in mid-event when we remove the job, we

  2620.         # can end up with one more event firing, so give it an extra

  2621.         # second to settle.

  2622.         time.sleep(1)

  2623.         self.waitUntilSettled()

  2624. 

  2625.         self.assertEqual(len(self.builds), 1, "One timer job")

  2626. 

  2627.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2628.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  2629.         self.waitUntilSettled()

  2630.         self.assertEqual(len(self.builds), 2, "One change plus one timer job")

  2631. 

  2632.         self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())

  2633.         self.waitUntilSettled()

  2634. 

  2635.         self.assertEqual(len(self.builds), 1, "One timer job remains")

  2636. 

  2637.         self.executor_server.release()

  2638.         self.waitUntilSettled()

  2639. 

  2640.     def test_new_patchset_dequeues_old_on_head(self):

  2641.         "Test that a new patchset causes the old to be dequeued (at head)"

  2642.         # D -> C (depends on B) -> B (depends on A) -> A -> M

  2643.         self.executor_server.hold_jobs_in_build = True

  2644.         M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')

  2645.         M.setMerged()

  2646.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2647.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  2648.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  2649.         D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')

  2650.         A.addApproval('Code-Review', 2)

  2651.         B.addApproval('Code-Review', 2)

  2652.         C.addApproval('Code-Review', 2)

  2653.         D.addApproval('Code-Review', 2)

  2654. 

  2655.         C.setDependsOn(B, 1)

  2656.         B.setDependsOn(A, 1)

  2657.         A.setDependsOn(M, 1)

  2658. 

  2659.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2660.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  2661.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  2662.         self.fake_gerrit.addEvent(D.addApproval('Approved', 1))

  2663.         self.waitUntilSettled()

  2664. 

  2665.         A.addPatchset()

  2666.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))

  2667.         self.waitUntilSettled()

  2668. 

  2669.         self.executor_server.hold_jobs_in_build = False

  2670.         self.executor_server.release()

  2671.         self.waitUntilSettled()

  2672. 

  2673.         self.assertEqual(A.data['status'], 'NEW')

  2674.         self.assertEqual(A.reported, 2)

  2675.         self.assertEqual(B.data['status'], 'NEW')

  2676.         self.assertEqual(B.reported, 2)

  2677.         self.assertEqual(C.data['status'], 'NEW')

  2678.         self.assertEqual(C.reported, 2)

  2679.         self.assertEqual(D.data['status'], 'MERGED')

  2680.         self.assertEqual(D.reported, 2)

  2681.         self.assertEqual(len(self.history), 7)

  2682. 

  2683.     def test_new_patchset_dequeues_old_without_dependents(self):

  2684.         "Test that a new patchset causes only the old to be dequeued"

  2685.         self.executor_server.hold_jobs_in_build = True

  2686.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2687.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  2688.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  2689.         A.addApproval('Code-Review', 2)

  2690.         B.addApproval('Code-Review', 2)

  2691.         C.addApproval('Code-Review', 2)

  2692. 

  2693.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  2694.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  2695.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2696.         self.waitUntilSettled()

  2697. 

  2698.         B.addPatchset()

  2699.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))

  2700.         self.waitUntilSettled()

  2701. 

  2702.         self.executor_server.hold_jobs_in_build = False

  2703.         self.executor_server.release()

  2704.         self.waitUntilSettled()

  2705. 

  2706.         self.assertEqual(A.data['status'], 'MERGED')

  2707.         self.assertEqual(A.reported, 2)

  2708.         self.assertEqual(B.data['status'], 'NEW')

  2709.         self.assertEqual(B.reported, 2)

  2710.         self.assertEqual(C.data['status'], 'MERGED')

  2711.         self.assertEqual(C.reported, 2)

  2712.         self.assertEqual(len(self.history), 9)

  2713. 

  2714.     def test_new_patchset_dequeues_old_independent_queue(self):

  2715.         "Test that a new patchset causes the old to be dequeued (independent)"

  2716.         self.executor_server.hold_jobs_in_build = True

  2717.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2718.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  2719.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  2720.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  2721.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  2722.         self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

  2723.         self.waitUntilSettled()

  2724. 

  2725.         B.addPatchset()

  2726.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))

  2727.         self.waitUntilSettled()

  2728. 

  2729.         self.executor_server.hold_jobs_in_build = False

  2730.         self.executor_server.release()

  2731.         self.waitUntilSettled()

  2732. 

  2733.         self.assertEqual(A.data['status'], 'NEW')

  2734.         self.assertEqual(A.reported, 1)

  2735.         self.assertEqual(B.data['status'], 'NEW')

  2736.         self.assertEqual(B.reported, 1)

  2737.         self.assertEqual(C.data['status'], 'NEW')

  2738.         self.assertEqual(C.reported, 1)

  2739.         self.assertEqual(len(self.history), 10)

  2740.         self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1)

  2741. 

  2742.     @simple_layout('layouts/noop-job.yaml')

  2743.     def test_noop_job(self):

  2744.         "Test that the internal noop job works"

  2745.         A = self.fake_gerrit.addFakeChange('org/noop-project', 'master', 'A')

  2746.         A.addApproval('Code-Review', 2)

  2747.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2748.         self.waitUntilSettled()

  2749. 

  2750.         self.assertEqual(len(self.gearman_server.getQueue()), 0)

  2751.         self.assertTrue(self.scheds.first.sched._areAllBuildsComplete())

  2752.         self.assertEqual(len(self.history), 0)

  2753.         self.assertEqual(A.data['status'], 'MERGED')

  2754.         self.assertEqual(A.reported, 2)

  2755. 

  2756.     @simple_layout('layouts/no-jobs-project.yaml')

  2757.     def test_no_job_project(self):

  2758.         "Test that reports with no jobs don't get sent"

  2759.         A = self.fake_gerrit.addFakeChange('org/no-jobs-project',

  2760.                                            'master', 'A')

  2761.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  2762.         self.waitUntilSettled()

  2763. 

  2764.         # Change wasn't reported to

  2765.         self.assertEqual(A.reported, False)

  2766. 

  2767.         # Check queue is empty afterwards

  2768.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  2769.         check_pipeline = tenant.layout.pipelines['check']

  2770.         items = check_pipeline.getAllItems()

  2771.         self.assertEqual(len(items), 0)

  2772. 

  2773.         self.assertEqual(len(self.history), 0)

  2774. 

  2775.     def test_zuul_refs(self):

  2776.         "Test that zuul refs exist and have the right changes"

  2777.         self.executor_server.hold_jobs_in_build = True

  2778.         M1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'M1')

  2779.         M1.setMerged()

  2780.         M2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'M2')

  2781.         M2.setMerged()

  2782. 

  2783.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  2784.         B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')

  2785.         C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')

  2786.         D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')

  2787.         A.addApproval('Code-Review', 2)

  2788.         B.addApproval('Code-Review', 2)

  2789.         C.addApproval('Code-Review', 2)

  2790.         D.addApproval('Code-Review', 2)

  2791.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2792.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  2793.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  2794.         self.fake_gerrit.addEvent(D.addApproval('Approved', 1))

  2795. 

  2796.         self.waitUntilSettled()

  2797.         self.executor_server.release('.*-merge')

  2798.         self.waitUntilSettled()

  2799.         self.executor_server.release('.*-merge')

  2800.         self.waitUntilSettled()

  2801.         self.executor_server.release('.*-merge')

  2802.         self.waitUntilSettled()

  2803.         self.executor_server.release('.*-merge')

  2804.         self.waitUntilSettled()

  2805. 

  2806.         a_build = b_build = c_build = d_build = None

  2807.         for x in self.builds:

  2808.             if x.parameters['zuul']['change'] == '3':

  2809.                 a_build = x

  2810.             elif x.parameters['zuul']['change'] == '4':

  2811.                 b_build = x

  2812.             elif x.parameters['zuul']['change'] == '5':

  2813.                 c_build = x

  2814.             elif x.parameters['zuul']['change'] == '6':

  2815.                 d_build = x

  2816.             if a_build and b_build and c_build and d_build:

  2817.                 break

  2818. 

  2819.         # should have a, not b, and should not be in project2

  2820.         self.assertTrue(a_build.hasChanges(A))

  2821.         self.assertFalse(a_build.hasChanges(B, M2))

  2822. 

  2823.         # should have a and b, and should not be in project2

  2824.         self.assertTrue(b_build.hasChanges(A, B))

  2825.         self.assertFalse(b_build.hasChanges(M2))

  2826. 

  2827.         # should have a and b in 1, c in 2

  2828.         self.assertTrue(c_build.hasChanges(A, B, C))

  2829.         self.assertFalse(c_build.hasChanges(D))

  2830. 

  2831.         # should have a and b in 1, c and d in 2

  2832.         self.assertTrue(d_build.hasChanges(A, B, C, D))

  2833. 

  2834.         self.executor_server.hold_jobs_in_build = False

  2835.         self.executor_server.release()

  2836.         self.waitUntilSettled()

  2837. 

  2838.         self.assertEqual(A.data['status'], 'MERGED')

  2839.         self.assertEqual(A.reported, 2)

  2840.         self.assertEqual(B.data['status'], 'MERGED')

  2841.         self.assertEqual(B.reported, 2)

  2842.         self.assertEqual(C.data['status'], 'MERGED')

  2843.         self.assertEqual(C.reported, 2)

  2844.         self.assertEqual(D.data['status'], 'MERGED')

  2845.         self.assertEqual(D.reported, 2)

  2846. 

  2847.     def test_rerun_on_error(self):

  2848.         "Test that if a worker fails to run a job, it is run again"

  2849.         self.executor_server.hold_jobs_in_build = True

  2850.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2851.         A.addApproval('Code-Review', 2)

  2852.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2853.         self.waitUntilSettled()

  2854. 

  2855.         self.builds[0].requeue = True

  2856.         self.executor_server.hold_jobs_in_build = False

  2857.         self.executor_server.release()

  2858.         self.waitUntilSettled()

  2859.         self.assertEqual(self.countJobResults(self.history, None), 1)

  2860.         self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)

  2861. 

  2862.     def test_statsd(self):

  2863.         "Test each of the statsd methods used in the scheduler"

  2864.         statsd = self.scheds.first.sched.statsd

  2865.         statsd.incr('test-incr')

  2866.         statsd.timing('test-timing', 3)

  2867.         statsd.gauge('test-gauge', 12)

  2868.         self.assertReportedStat('test-incr', '1', 'c')

  2869.         self.assertReportedStat('test-timing', '3', 'ms')

  2870.         self.assertReportedStat('test-gauge', '12', 'g')

  2871. 

  2872.         # test key normalization

  2873.         statsd.extra_keys = {'hostname': '1_2_3_4'}

  2874. 

  2875.         statsd.incr('hostname-incr.{hostname}.{fake}', fake='1:2')

  2876.         statsd.timing('hostname-timing.{hostname}.{fake}', 3, fake='1:2')

  2877.         statsd.gauge('hostname-gauge.{hostname}.{fake}', 12, fake='1:2')

  2878.         self.assertReportedStat('hostname-incr.1_2_3_4.1_2', '1', 'c')

  2879.         self.assertReportedStat('hostname-timing.1_2_3_4.1_2', '3', 'ms')

  2880.         self.assertReportedStat('hostname-gauge.1_2_3_4.1_2', '12', 'g')

  2881. 

  2882.     def test_statsd_conflict(self):

  2883.         statsd = self.scheds.first.sched.statsd

  2884.         statsd.gauge('test-gauge', 12)

  2885.         # since test-gauge is already a value, we can't make

  2886.         # subvalues.  Test the assert works.

  2887.         statsd.gauge('test-gauge.1_2_3_4', 12)

  2888.         self.assertReportedStat('test-gauge', '12', 'g')

  2889.         self.assertRaises(Exception, self.assertReportedStat,

  2890.                           'test-gauge.1_2_3_4', '12', 'g')

  2891. 

  2892.     def test_stuck_job_cleanup(self):

  2893.         "Test that pending jobs are cleaned up if removed from layout"

  2894. 

  2895.         # We want to hold the project-merge job that the fake change enqueues

  2896.         self.gearman_server.hold_jobs_in_queue = True

  2897.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2898.         A.addApproval('Code-Review', 2)

  2899.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2900.         self.waitUntilSettled()

  2901.         # The assertion is that we have one job in the queue, project-merge

  2902.         self.assertEqual(len(self.gearman_server.getQueue()), 1)

  2903. 

  2904.         self.commitConfigUpdate('common-config', 'layouts/no-jobs.yaml')

  2905.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  2906.         self.waitUntilSettled()

  2907. 

  2908.         self.gearman_server.release('gate-noop')

  2909.         self.waitUntilSettled()

  2910.         # asserting that project-merge is removed from queue

  2911.         self.assertEqual(len(self.gearman_server.getQueue()), 0)

  2912.         self.assertTrue(self.scheds.first.sched._areAllBuildsComplete())

  2913. 

  2914.         self.assertEqual(len(self.history), 1)

  2915.         self.assertEqual(self.history[0].name, 'gate-noop')

  2916.         self.assertEqual(self.history[0].result, 'SUCCESS')

  2917. 

  2918.     def test_file_head(self):

  2919.         # This is a regression test for an observed bug.  A change

  2920.         # with a file named "HEAD" in the root directory of the repo

  2921.         # was processed by a merger.  It then was unable to reset the

  2922.         # repo because of:

  2923.         #   GitCommandError: 'git reset --hard HEAD' returned

  2924.         #       with exit code 128

  2925.         #   stderr: 'fatal: ambiguous argument 'HEAD': both revision

  2926.         #       and filename

  2927.         #   Use '--' to separate filenames from revisions'

  2928. 

  2929.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2930.         A.addPatchset({'HEAD': ''})

  2931.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  2932. 

  2933.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))

  2934.         self.waitUntilSettled()

  2935. 

  2936.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  2937.         self.waitUntilSettled()

  2938. 

  2939.         self.assertIn('Build succeeded', A.messages[0])

  2940.         self.assertIn('Build succeeded', B.messages[0])

  2941. 

  2942.     def test_file_jobs(self):

  2943.         "Test that file jobs run only when appropriate"

  2944.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  2945.         A.addPatchset({'pip-requires': 'foo'})

  2946.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  2947.         A.addApproval('Code-Review', 2)

  2948.         B.addApproval('Code-Review', 2)

  2949.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  2950.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  2951.         self.waitUntilSettled()

  2952. 

  2953.         testfile_jobs = [x for x in self.history

  2954.                          if x.name == 'project-testfile']

  2955. 

  2956.         self.assertEqual(len(testfile_jobs), 1)

  2957.         self.assertEqual(testfile_jobs[0].changes, '1,2')

  2958.         self.assertEqual(A.data['status'], 'MERGED')

  2959.         self.assertEqual(A.reported, 2)

  2960.         self.assertEqual(B.data['status'], 'MERGED')

  2961.         self.assertEqual(B.reported, 2)

  2962. 

  2963.     def _test_irrelevant_files_jobs(self, should_skip):

  2964.         "Test that jobs with irrelevant-files filter run only when appropriate"

  2965.         if should_skip:

  2966.             files = {'ignoreme': 'ignored\n'}

  2967.         else:

  2968.             files = {'respectme': 'please!\n'}

  2969. 

  2970.         change = self.fake_gerrit.addFakeChange('org/project',

  2971.                                                 'master',

  2972.                                                 'test irrelevant-files',

  2973.                                                 files=files)

  2974.         self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))

  2975.         self.waitUntilSettled()

  2976. 

  2977.         tested_change_ids = [x.changes[0] for x in self.history

  2978.                              if x.name == 'project-test-irrelevant-files']

  2979. 

  2980.         if should_skip:

  2981.             self.assertEqual([], tested_change_ids)

  2982.         else:

  2983.             self.assertIn(change.data['number'], tested_change_ids)

  2984. 

  2985.     @simple_layout('layouts/irrelevant-files.yaml')

  2986.     def test_irrelevant_files_match_skips_job(self):

  2987.         self._test_irrelevant_files_jobs(should_skip=True)

  2988. 

  2989.     @simple_layout('layouts/irrelevant-files.yaml')

  2990.     def test_irrelevant_files_no_match_runs_job(self):

  2991.         self._test_irrelevant_files_jobs(should_skip=False)

  2992. 

  2993.     @simple_layout('layouts/inheritance.yaml')

  2994.     def test_inherited_jobs_keep_matchers(self):

  2995.         files = {'ignoreme': 'ignored\n'}

  2996. 

  2997.         change = self.fake_gerrit.addFakeChange('org/project',

  2998.                                                 'master',

  2999.                                                 'test irrelevant-files',

  3000.                                                 files=files)

  3001.         self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))

  3002.         self.waitUntilSettled()

  3003. 

  3004.         run_jobs = set([build.name for build in self.history])

  3005. 

  3006.         self.assertEqual(set(['project-test-nomatch-starts-empty',

  3007.                               'project-test-nomatch-starts-full']), run_jobs)

  3008. 

  3009.     @simple_layout('layouts/job-vars.yaml')

  3010.     def test_inherited_job_variables(self):

  3011.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  3012.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  3013.         self.waitUntilSettled()

  3014.         self.assertHistory([

  3015.             dict(name='parentjob', result='SUCCESS'),

  3016.             dict(name='child1', result='SUCCESS'),

  3017.             dict(name='child2', result='SUCCESS'),

  3018.             dict(name='child3', result='SUCCESS'),

  3019.             dict(name='override_project_var', result='SUCCESS'),

  3020.             dict(name='job_from_template1', result='SUCCESS'),

  3021.             dict(name='job_from_template2', result='SUCCESS'),

  3022.         ], ordered=False)

  3023.         j = self.getJobFromHistory('parentjob')

  3024.         rp = set([p['name'] for p in j.parameters['projects']])

  3025.         self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project')

  3026.         self.assertEqual(j.parameters['vars']['template_var1'],

  3027.                          'set_in_template1')

  3028.         self.assertEqual(j.parameters['vars']['template_var2'],

  3029.                          'set_in_template2')

  3030.         self.assertEqual(j.parameters['vars']['override'], 0)

  3031.         self.assertEqual(j.parameters['vars']['child1override'], 0)

  3032.         self.assertEqual(j.parameters['vars']['parent'], 0)

  3033.         self.assertEqual(j.parameters['vars']['deep']['override'], 0)

  3034.         self.assertFalse('child1' in j.parameters['vars'])

  3035.         self.assertFalse('child2' in j.parameters['vars'])

  3036.         self.assertFalse('child3' in j.parameters['vars'])

  3037.         self.assertEqual(rp, set(['org/project', 'org/project0',

  3038.                                   'org/project0']))

  3039.         j = self.getJobFromHistory('child1')

  3040.         rp = set([p['name'] for p in j.parameters['projects']])

  3041.         self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project')

  3042.         self.assertEqual(j.parameters['vars']['override'], 1)

  3043.         self.assertEqual(j.parameters['vars']['child1override'], 1)

  3044.         self.assertEqual(j.parameters['vars']['parent'], 0)

  3045.         self.assertEqual(j.parameters['vars']['child1'], 1)

  3046.         self.assertEqual(j.parameters['vars']['deep']['override'], 1)

  3047.         self.assertFalse('child2' in j.parameters['vars'])

  3048.         self.assertFalse('child3' in j.parameters['vars'])

  3049.         self.assertEqual(rp, set(['org/project', 'org/project0',

  3050.                                   'org/project1']))

  3051.         j = self.getJobFromHistory('child2')

  3052.         self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project')

  3053.         rp = set([p['name'] for p in j.parameters['projects']])

  3054.         self.assertEqual(j.parameters['vars']['override'], 2)

  3055.         self.assertEqual(j.parameters['vars']['child1override'], 0)

  3056.         self.assertEqual(j.parameters['vars']['parent'], 0)

  3057.         self.assertEqual(j.parameters['vars']['deep']['override'], 2)

  3058.         self.assertFalse('child1' in j.parameters['vars'])

  3059.         self.assertEqual(j.parameters['vars']['child2'], 2)

  3060.         self.assertFalse('child3' in j.parameters['vars'])

  3061.         self.assertEqual(rp, set(['org/project', 'org/project0',

  3062.                                   'org/project2']))

  3063.         j = self.getJobFromHistory('child3')

  3064.         self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project')

  3065.         rp = set([p['name'] for p in j.parameters['projects']])

  3066.         self.assertEqual(j.parameters['vars']['override'], 3)

  3067.         self.assertEqual(j.parameters['vars']['child1override'], 0)

  3068.         self.assertEqual(j.parameters['vars']['parent'], 0)

  3069.         self.assertEqual(j.parameters['vars']['deep']['override'], 3)

  3070.         self.assertFalse('child1' in j.parameters['vars'])

  3071.         self.assertFalse('child2' in j.parameters['vars'])

  3072.         self.assertEqual(j.parameters['vars']['child3'], 3)

  3073.         self.assertEqual(rp, set(['org/project', 'org/project0',

  3074.                                   'org/project3']))

  3075.         j = self.getJobFromHistory('override_project_var')

  3076.         self.assertEqual(j.parameters['vars']['project_var'],

  3077.                          'override_in_job')

  3078. 

  3079.     @simple_layout('layouts/job-variants.yaml')

  3080.     def test_job_branch_variants(self):

  3081.         self.create_branch('org/project', 'stable/diablo')

  3082.         self.fake_gerrit.addEvent(

  3083.             self.fake_gerrit.getFakeBranchCreatedEvent(

  3084.                 'org/project', 'stable/diablo'))

  3085.         self.create_branch('org/project', 'stable/essex')

  3086.         self.fake_gerrit.addEvent(

  3087.             self.fake_gerrit.getFakeBranchCreatedEvent(

  3088.                 'org/project', 'stable/essex'))

  3089.         self.waitUntilSettled()

  3090. 

  3091.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  3092.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  3093.         self.waitUntilSettled()

  3094. 

  3095.         B = self.fake_gerrit.addFakeChange('org/project', 'stable/diablo', 'B')

  3096.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  3097.         self.waitUntilSettled()

  3098. 

  3099.         C = self.fake_gerrit.addFakeChange('org/project', 'stable/essex', 'C')

  3100.         self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

  3101.         self.waitUntilSettled()

  3102.         self.assertHistory([

  3103.             dict(name='python27', result='SUCCESS'),

  3104.             dict(name='python27', result='SUCCESS'),

  3105.             dict(name='python27', result='SUCCESS'),

  3106.         ])

  3107. 

  3108.         p = self.history[0].parameters

  3109.         self.assertEqual(p['timeout'], 40)

  3110.         self.assertEqual(len(p['nodes']), 1)

  3111.         self.assertEqual(p['nodes'][0]['label'], 'new')

  3112.         self.assertEqual([x['path'] for x in p['pre_playbooks']],

  3113.                          ['base-pre', 'py27-pre'])

  3114.         self.assertEqual([x['path'] for x in p['post_playbooks']],

  3115.                          ['py27-post-a', 'py27-post-b', 'base-post'])

  3116.         self.assertEqual([x['path'] for x in p['playbooks']],

  3117.                          ['playbooks/python27.yaml'])

  3118. 

  3119.         p = self.history[1].parameters

  3120.         self.assertEqual(p['timeout'], 50)

  3121.         self.assertEqual(len(p['nodes']), 1)

  3122.         self.assertEqual(p['nodes'][0]['label'], 'old')

  3123.         self.assertEqual([x['path'] for x in p['pre_playbooks']],

  3124.                          ['base-pre', 'py27-pre', 'py27-diablo-pre'])

  3125.         self.assertEqual([x['path'] for x in p['post_playbooks']],

  3126.                          ['py27-diablo-post', 'py27-post-a', 'py27-post-b',

  3127.                           'base-post'])

  3128.         self.assertEqual([x['path'] for x in p['playbooks']],

  3129.                          ['py27-diablo'])

  3130. 

  3131.         p = self.history[2].parameters

  3132.         self.assertEqual(p['timeout'], 40)

  3133.         self.assertEqual(len(p['nodes']), 1)

  3134.         self.assertEqual(p['nodes'][0]['label'], 'new')

  3135.         self.assertEqual([x['path'] for x in p['pre_playbooks']],

  3136.                          ['base-pre', 'py27-pre', 'py27-essex-pre'])

  3137.         self.assertEqual([x['path'] for x in p['post_playbooks']],

  3138.                          ['py27-essex-post', 'py27-post-a', 'py27-post-b',

  3139.                           'base-post'])

  3140.         self.assertEqual([x['path'] for x in p['playbooks']],

  3141.                          ['playbooks/python27.yaml'])

  3142. 

  3143.     @simple_layout("layouts/no-run.yaml")

  3144.     def test_job_without_run(self):

  3145.         "Test that a job without a run playbook errors"

  3146.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  3147.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  3148.         self.waitUntilSettled()

  3149.         self.assertIn('Job base does not specify a run playbook',

  3150.                       A.messages[-1])

  3151. 

  3152.     def test_queue_names(self):

  3153.         "Test shared change queue names"

  3154.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  3155.         (trusted, project1) = tenant.getProject('org/project1')

  3156.         (trusted, project2) = tenant.getProject('org/project2')

  3157.         q1 = tenant.layout.pipelines['gate'].getQueue(project1)

  3158.         q2 = tenant.layout.pipelines['gate'].getQueue(project2)

  3159.         self.assertEqual(q1.name, 'integrated')

  3160.         self.assertEqual(q2.name, 'integrated')

  3161. 

  3162.     @simple_layout("layouts/template-queue.yaml")

  3163.     def test_template_queue(self):

  3164.         "Test a shared queue can be constructed from a project-template"

  3165.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  3166.         (trusted, project1) = tenant.getProject('org/project1')

  3167.         (trusted, project2) = tenant.getProject('org/project2')

  3168.         q1 = tenant.layout.pipelines['gate'].getQueue(project1)

  3169.         q2 = tenant.layout.pipelines['gate'].getQueue(project2)

  3170.         self.assertEqual(q1.name, 'integrated')

  3171.         self.assertEqual(q2.name, 'integrated')

  3172. 

  3173.     @simple_layout("layouts/regex-template-queue.yaml")

  3174.     def test_regex_template_queue(self):

  3175.         "Test a shared queue can be constructed from a regex project-template"

  3176.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  3177.         (trusted, project1) = tenant.getProject('org/project1')

  3178.         (trusted, project2) = tenant.getProject('org/project2')

  3179.         q1 = tenant.layout.pipelines['gate'].getQueue(project1)

  3180.         q2 = tenant.layout.pipelines['gate'].getQueue(project2)

  3181.         self.assertEqual(q1.name, 'integrated')

  3182.         self.assertEqual(q2.name, 'integrated')

  3183. 

  3184.     @simple_layout("layouts/regex-queue.yaml")

  3185.     def test_regex_queue(self):

  3186.         "Test a shared queue can be constructed from a regex project"

  3187.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  3188.         (trusted, project1) = tenant.getProject('org/project1')

  3189.         (trusted, project2) = tenant.getProject('org/project2')

  3190.         q1 = tenant.layout.pipelines['gate'].getQueue(project1)

  3191.         q2 = tenant.layout.pipelines['gate'].getQueue(project2)

  3192.         self.assertEqual(q1.name, 'integrated')

  3193.         self.assertEqual(q2.name, 'integrated')

  3194. 

  3195.     def test_queue_precedence(self):

  3196.         "Test that queue precedence works"

  3197. 

  3198.         self.gearman_server.hold_jobs_in_queue = True

  3199.         self.executor_server.hold_jobs_in_build = True

  3200.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  3201.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  3202.         A.addApproval('Code-Review', 2)

  3203.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  3204. 

  3205.         self.waitUntilSettled()

  3206.         self.gearman_server.hold_jobs_in_queue = False

  3207.         self.gearman_server.release()

  3208.         self.waitUntilSettled()

  3209. 

  3210.         # Run one build at a time to ensure non-race order:

  3211.         self.orderedRelease()

  3212.         self.executor_server.hold_jobs_in_build = False

  3213.         self.waitUntilSettled()

  3214. 

  3215.         self.log.debug(self.history)

  3216.         self.assertEqual(self.history[0].pipeline, 'gate')

  3217.         self.assertEqual(self.history[1].pipeline, 'check')

  3218.         self.assertEqual(self.history[2].pipeline, 'gate')

  3219.         self.assertEqual(self.history[3].pipeline, 'gate')

  3220.         self.assertEqual(self.history[4].pipeline, 'check')

  3221.         self.assertEqual(self.history[5].pipeline, 'check')

  3222. 

  3223.     def test_reconfigure_merge(self):

  3224.         """Test that two reconfigure events are merged"""

  3225. 

  3226.         tenant = self.scheds.first.sched.abide.tenants['tenant-one']

  3227.         (trusted, project) = tenant.getProject('org/project')

  3228. 

  3229.         self.scheds.first.sched.run_handler_lock.acquire()

  3230.         self.assertEqual(

  3231.             self.scheds.first.sched.management_event_queue.qsize(), 0)

  3232. 

  3233.         self.scheds.first.sched.reconfigureTenant(tenant, project, None)

  3234.         self.assertEqual(

  3235.             self.scheds.first.sched.management_event_queue.qsize(), 1)

  3236. 

  3237.         self.scheds.first.sched.reconfigureTenant(tenant, project, None)

  3238.         # The second event should have been combined with the first

  3239.         # so we should still only have one entry.

  3240.         self.assertEqual(

  3241.             self.scheds.first.sched.management_event_queue.qsize(), 1)

  3242. 

  3243.         self.scheds.first.sched.run_handler_lock.release()

  3244.         self.waitUntilSettled()

  3245. 

  3246.         self.assertEqual(

  3247.             self.scheds.first.sched.management_event_queue.qsize(), 0)

  3248. 

  3249.     def test_live_reconfiguration(self):

  3250.         "Test that live reconfiguration works"

  3251.         self.executor_server.hold_jobs_in_build = True

  3252.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  3253.         A.addApproval('Code-Review', 2)

  3254.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  3255.         self.waitUntilSettled()

  3256. 

  3257.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3258.         self.waitUntilSettled()

  3259. 

  3260.         self.executor_server.hold_jobs_in_build = False

  3261.         self.executor_server.release()

  3262.         self.waitUntilSettled()

  3263.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  3264.                          'SUCCESS')

  3265.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  3266.                          'SUCCESS')

  3267.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  3268.                          'SUCCESS')

  3269.         self.assertEqual(A.data['status'], 'MERGED')

  3270.         self.assertEqual(A.reported, 2)

  3271. 

  3272.     def test_live_reconfiguration_command_socket(self):

  3273.         "Test that live reconfiguration via command socket works"

  3274. 

  3275.         # record previous tenant reconfiguration time, which may not be set

  3276.         old = self.scheds.first.sched.tenant_last_reconfigured\

  3277.             .get('tenant-one', 0)

  3278.         self.waitUntilSettled()

  3279. 

  3280.         command_socket = self.config.get('scheduler', 'command_socket')

  3281.         with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:

  3282.             s.connect(command_socket)

  3283.             s.sendall('full-reconfigure\n'.encode('utf8'))

  3284. 

  3285.         # Wait for full reconfiguration. Note that waitUntilSettled is not

  3286.         # reliable here because the reconfigure event may arrive in the

  3287.         # event queue after waitUntilSettled.

  3288.         start = time.time()

  3289.         while True:

  3290.             if time.time() - start > 15:

  3291.                 raise Exception("Timeout waiting for full reconfiguration")

  3292.             new = self.scheds.first.sched.tenant_last_reconfigured\

  3293.                 .get('tenant-one', 0)

  3294.             if old < new:

  3295.                 break

  3296.             else:

  3297.                 time.sleep(0)

  3298. 

  3299.     def test_live_reconfiguration_abort(self):

  3300.         # Raise an exception during reconfiguration and verify we

  3301.         # still function.

  3302.         self.executor_server.hold_jobs_in_build = True

  3303.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  3304.         A.addApproval('Code-Review', 2)

  3305.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  3306.         self.waitUntilSettled()

  3307. 

  3308.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  3309.         pipeline = tenant.layout.pipelines['gate']

  3310.         change = pipeline.getAllItems()[0].change

  3311.         # Set this to an invalid value to cause an exception during

  3312.         # reconfiguration.

  3313.         change.branch = None

  3314. 

  3315.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3316.         self.waitUntilSettled()

  3317. 

  3318.         self.executor_server.hold_jobs_in_build = False

  3319.         self.executor_server.release()

  3320. 

  3321.         self.waitUntilSettled()

  3322.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  3323.                          'ABORTED')

  3324.         self.assertEqual(A.data['status'], 'NEW')

  3325.         # The final report fails because of the invalid value set above.

  3326.         self.assertEqual(A.reported, 1)

  3327. 

  3328.     def test_live_reconfiguration_merge_conflict(self):

  3329.         # A real-world bug: a change in a gate queue has a merge

  3330.         # conflict and a job is added to its project while it's

  3331.         # sitting in the queue.  The job gets added to the change and

  3332.         # enqueued and the change gets stuck.

  3333.         self.executor_server.hold_jobs_in_build = True

  3334. 

  3335.         # This change is fine.  It's here to stop the queue long

  3336.         # enough for the next change to be subject to the

  3337.         # reconfiguration, as well as to provide a conflict for the

  3338.         # next change.  This change will succeed and merge.

  3339.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  3340.         A.addPatchset({'conflict': 'A'})

  3341.         A.addApproval('Code-Review', 2)

  3342. 

  3343.         # This change will be in merge conflict.  During the

  3344.         # reconfiguration, we will add a job.  We want to make sure

  3345.         # that doesn't cause it to get stuck.

  3346.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  3347.         B.addPatchset({'conflict': 'B'})

  3348.         B.addApproval('Code-Review', 2)

  3349. 

  3350.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  3351.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  3352. 

  3353.         self.waitUntilSettled()

  3354. 

  3355.         # No jobs have run yet

  3356.         self.assertEqual(A.data['status'], 'NEW')

  3357.         self.assertEqual(A.reported, 1)

  3358.         self.assertEqual(B.data['status'], 'NEW')

  3359.         self.assertEqual(len(self.history), 0)

  3360. 

  3361.         # Add the "project-test3" job.

  3362.         self.commitConfigUpdate('common-config',

  3363.                                 'layouts/live-reconfiguration-add-job.yaml')

  3364.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3365.         self.waitUntilSettled()

  3366. 

  3367.         self.executor_server.hold_jobs_in_build = False

  3368.         self.executor_server.release()

  3369.         self.waitUntilSettled()

  3370. 

  3371.         self.assertEqual(A.data['status'], 'MERGED')

  3372.         self.assertEqual(A.reported, 2)

  3373.         self.assertEqual(B.data['status'], 'NEW')

  3374.         self.assertIn('Merge Failed', B.messages[-1])

  3375.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  3376.                          'SUCCESS')

  3377.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  3378.                          'SUCCESS')

  3379.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  3380.                          'SUCCESS')

  3381.         self.assertEqual(self.getJobFromHistory('project-test3').result,

  3382.                          'SUCCESS')

  3383.         self.assertEqual(len(self.history), 4)

  3384. 

  3385.     def test_live_reconfiguration_failed_root(self):

  3386.         # An extrapolation of test_live_reconfiguration_merge_conflict

  3387.         # that tests a job added to a job tree with a failed root does

  3388.         # not run.

  3389.         self.executor_server.hold_jobs_in_build = True

  3390. 

  3391.         # This change is fine.  It's here to stop the queue long

  3392.         # enough for the next change to be subject to the

  3393.         # reconfiguration.  This change will succeed and merge.

  3394.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  3395.         A.addPatchset({'conflict': 'A'})

  3396.         A.addApproval('Code-Review', 2)

  3397.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  3398.         self.waitUntilSettled()

  3399.         self.executor_server.release('.*-merge')

  3400.         self.waitUntilSettled()

  3401. 

  3402.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  3403.         self.executor_server.failJob('project-merge', B)

  3404.         B.addApproval('Code-Review', 2)

  3405.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  3406.         self.waitUntilSettled()

  3407. 

  3408.         self.executor_server.release('.*-merge')

  3409.         self.waitUntilSettled()

  3410. 

  3411.         # Both -merge jobs have run, but no others.

  3412.         self.assertEqual(A.data['status'], 'NEW')

  3413.         self.assertEqual(A.reported, 1)

  3414.         self.assertEqual(B.data['status'], 'NEW')

  3415.         self.assertEqual(B.reported, 1)

  3416.         self.assertEqual(self.history[0].result, 'SUCCESS')

  3417.         self.assertEqual(self.history[0].name, 'project-merge')

  3418.         self.assertEqual(self.history[1].result, 'FAILURE')

  3419.         self.assertEqual(self.history[1].name, 'project-merge')

  3420.         self.assertEqual(len(self.history), 2)

  3421. 

  3422.         # Add the "project-test3" job.

  3423.         self.commitConfigUpdate('common-config',

  3424.                                 'layouts/live-reconfiguration-add-job.yaml')

  3425.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3426.         self.waitUntilSettled()

  3427. 

  3428.         self.executor_server.hold_jobs_in_build = False

  3429.         self.executor_server.release()

  3430.         self.waitUntilSettled()

  3431. 

  3432.         self.assertEqual(A.data['status'], 'MERGED')

  3433.         self.assertEqual(A.reported, 2)

  3434.         self.assertEqual(B.data['status'], 'NEW')

  3435.         self.assertEqual(B.reported, 2)

  3436.         self.assertEqual(self.history[0].result, 'SUCCESS')

  3437.         self.assertEqual(self.history[0].name, 'project-merge')

  3438.         self.assertEqual(self.history[1].result, 'FAILURE')

  3439.         self.assertEqual(self.history[1].name, 'project-merge')

  3440.         self.assertEqual(self.history[2].result, 'SUCCESS')

  3441.         self.assertEqual(self.history[3].result, 'SUCCESS')

  3442.         self.assertEqual(self.history[4].result, 'SUCCESS')

  3443.         self.assertEqual(len(self.history), 5)

  3444. 

  3445.     def test_live_reconfiguration_failed_job(self):

  3446.         # Test that a change with a removed failing job does not

  3447.         # disrupt reconfiguration.  If a change has a failed job and

  3448.         # that job is removed during a reconfiguration, we observed a

  3449.         # bug where the code to re-set build statuses would run on

  3450.         # that build and raise an exception because the job no longer

  3451.         # existed.

  3452.         self.executor_server.hold_jobs_in_build = True

  3453. 

  3454.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  3455. 

  3456.         # This change will fail and later be removed by the reconfiguration.

  3457.         self.executor_server.failJob('project-test1', A)

  3458. 

  3459.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  3460.         self.waitUntilSettled()

  3461.         self.executor_server.release('.*-merge')

  3462.         self.waitUntilSettled()

  3463.         self.executor_server.release('project-test1')

  3464.         self.waitUntilSettled()

  3465. 

  3466.         self.assertEqual(A.data['status'], 'NEW')

  3467.         self.assertEqual(A.reported, 0)

  3468. 

  3469.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  3470.                          'SUCCESS')

  3471.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  3472.                          'FAILURE')

  3473.         self.assertEqual(len(self.history), 2)

  3474. 

  3475.         # Remove the test1 job.

  3476.         self.commitConfigUpdate('common-config',

  3477.                                 'layouts/live-reconfiguration-failed-job.yaml')

  3478.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3479.         self.waitUntilSettled()

  3480. 

  3481.         self.executor_server.hold_jobs_in_build = False

  3482.         self.executor_server.release()

  3483.         self.waitUntilSettled()

  3484. 

  3485.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  3486.                          'SUCCESS')

  3487.         self.assertEqual(self.getJobFromHistory('project-testfile').result,

  3488.                          'SUCCESS')

  3489.         self.assertEqual(len(self.history), 4)

  3490. 

  3491.         self.assertEqual(A.data['status'], 'NEW')

  3492.         self.assertEqual(A.reported, 1)

  3493.         self.assertIn('Build succeeded', A.messages[0])

  3494.         # Ensure the removed job was not included in the report.

  3495.         self.assertNotIn('project-test1', A.messages[0])

  3496. 

  3497.     def test_live_reconfiguration_shared_queue(self):

  3498.         # Test that a change with a failing job which was removed from

  3499.         # this project but otherwise still exists in the system does

  3500.         # not disrupt reconfiguration.

  3501. 

  3502.         self.executor_server.hold_jobs_in_build = True

  3503. 

  3504.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  3505. 

  3506.         self.executor_server.failJob('project1-project2-integration', A)

  3507. 

  3508.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  3509.         self.waitUntilSettled()

  3510.         self.executor_server.release('.*-merge')

  3511.         self.waitUntilSettled()

  3512.         self.executor_server.release('project1-project2-integration')

  3513.         self.waitUntilSettled()

  3514. 

  3515.         self.assertEqual(A.data['status'], 'NEW')

  3516.         self.assertEqual(A.reported, 0)

  3517. 

  3518.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  3519.                          'SUCCESS')

  3520.         self.assertEqual(self.getJobFromHistory(

  3521.             'project1-project2-integration').result, 'FAILURE')

  3522.         self.assertEqual(len(self.history), 2)

  3523. 

  3524.         # Remove the integration job.

  3525.         self.commitConfigUpdate(

  3526.             'common-config',

  3527.             'layouts/live-reconfiguration-shared-queue.yaml')

  3528.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3529.         self.waitUntilSettled()

  3530. 

  3531.         self.executor_server.hold_jobs_in_build = False

  3532.         self.executor_server.release()

  3533.         self.waitUntilSettled()

  3534. 

  3535.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  3536.                          'SUCCESS')

  3537.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  3538.                          'SUCCESS')

  3539.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  3540.                          'SUCCESS')

  3541.         self.assertEqual(self.getJobFromHistory(

  3542.             'project1-project2-integration').result, 'FAILURE')

  3543.         self.assertEqual(len(self.history), 4)

  3544. 

  3545.         self.assertEqual(A.data['status'], 'NEW')

  3546.         self.assertEqual(A.reported, 1)

  3547.         self.assertIn('Build succeeded', A.messages[0])

  3548.         # Ensure the removed job was not included in the report.

  3549.         self.assertNotIn('project1-project2-integration', A.messages[0])

  3550. 

  3551.     def test_live_reconfiguration_shared_queue_removed(self):

  3552.         # Test that changes in a shared queue survive a change of the

  3553.         # queue during reconfiguration. This is a regression test

  3554.         # for the dependent pipeline manager.

  3555.         self.executor_server.hold_jobs_in_build = True

  3556. 

  3557.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  3558.         B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')

  3559.         A.addApproval('Code-Review', 2)

  3560.         B.addApproval('Code-Review', 2)

  3561. 

  3562.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  3563.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  3564.         self.waitUntilSettled()

  3565. 

  3566.         # Remove the integration job.

  3567.         self.commitConfigUpdate(

  3568.             'common-config',

  3569.             'layouts/live-reconfiguration-shared-queue-removed.yaml')

  3570.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3571.         self.waitUntilSettled()

  3572. 

  3573.         self.executor_server.hold_jobs_in_build = False

  3574.         self.executor_server.release()

  3575.         self.waitUntilSettled()

  3576. 

  3577.         self.assertEqual(A.data['status'], 'MERGED')

  3578.         self.assertEqual(B.data['status'], 'MERGED')

  3579. 

  3580.     def test_double_live_reconfiguration_shared_queue(self):

  3581.         # This was a real-world regression.  A change is added to

  3582.         # gate; a reconfigure happens, a second change which depends

  3583.         # on the first is added, and a second reconfiguration happens.

  3584.         # Ensure that both changes merge.

  3585. 

  3586.         # A failure may indicate incorrect caching or cleaning up of

  3587.         # references during a reconfiguration.

  3588.         self.executor_server.hold_jobs_in_build = True

  3589. 

  3590.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  3591.         B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')

  3592.         B.setDependsOn(A, 1)

  3593.         A.addApproval('Code-Review', 2)

  3594.         B.addApproval('Code-Review', 2)

  3595. 

  3596.         # Add the parent change.

  3597.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  3598.         self.waitUntilSettled()

  3599.         self.executor_server.release('.*-merge')

  3600.         self.waitUntilSettled()

  3601. 

  3602.         # Reconfigure (with only one change in the pipeline).

  3603.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3604.         self.waitUntilSettled()

  3605. 

  3606.         # Add the child change.

  3607.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  3608.         self.waitUntilSettled()

  3609.         self.executor_server.release('.*-merge')

  3610.         self.waitUntilSettled()

  3611. 

  3612.         # Reconfigure (with both in the pipeline).

  3613.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3614.         self.waitUntilSettled()

  3615. 

  3616.         self.executor_server.hold_jobs_in_build = False

  3617.         self.executor_server.release()

  3618.         self.waitUntilSettled()

  3619. 

  3620.         self.assertEqual(len(self.history), 8)

  3621. 

  3622.         self.assertEqual(A.data['status'], 'MERGED')

  3623.         self.assertEqual(A.reported, 2)

  3624.         self.assertEqual(B.data['status'], 'MERGED')

  3625.         self.assertEqual(B.reported, 2)

  3626. 

  3627.     def test_live_reconfiguration_del_project(self):

  3628.         # Test project deletion from layout

  3629.         # while changes are enqueued

  3630. 

  3631.         self.executor_server.hold_jobs_in_build = True

  3632.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  3633.         B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')

  3634.         C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')

  3635. 

  3636.         # A Depends-On: B

  3637.         A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (

  3638.             A.subject, B.data['id'])

  3639.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  3640. 

  3641.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  3642.         self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

  3643.         self.waitUntilSettled()

  3644.         self.executor_server.release('.*-merge')

  3645.         self.waitUntilSettled()

  3646.         self.assertEqual(len(self.builds), 5)

  3647. 

  3648.         # This layout defines only org/project, not org/project1

  3649.         self.commitConfigUpdate(

  3650.             'common-config',

  3651.             'layouts/live-reconfiguration-del-project.yaml')

  3652.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3653.         self.waitUntilSettled()

  3654. 

  3655.         # Builds for C aborted, builds for A succeed,

  3656.         # and have change B applied ahead

  3657.         job_c = self.getJobFromHistory('project-test1')

  3658.         self.assertEqual(job_c.changes, '3,1')

  3659.         self.assertEqual(job_c.result, 'ABORTED')

  3660. 

  3661.         self.executor_server.hold_jobs_in_build = False

  3662.         self.executor_server.release()

  3663.         self.waitUntilSettled()

  3664. 

  3665.         self.assertEqual(

  3666.             self.getJobFromHistory('project-test1', 'org/project').changes,

  3667.             '2,1 1,1')

  3668. 

  3669.         self.assertEqual(A.data['status'], 'NEW')

  3670.         self.assertEqual(B.data['status'], 'NEW')

  3671.         self.assertEqual(C.data['status'], 'NEW')

  3672.         self.assertEqual(A.reported, 1)

  3673.         self.assertEqual(B.reported, 0)

  3674.         self.assertEqual(C.reported, 0)

  3675. 

  3676.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  3677.         self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)

  3678.         self.assertIn('Build succeeded', A.messages[0])

  3679. 

  3680.     @simple_layout("layouts/reconfigure-failed-head.yaml")

  3681.     def test_live_reconfiguration_failed_change_at_head(self):

  3682.         # Test that if we reconfigure with a failed change at head,

  3683.         # that the change behind it isn't reparented onto it.

  3684. 

  3685.         self.executor_server.hold_jobs_in_build = True

  3686.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  3687.         A.addApproval('Code-Review', 2)

  3688.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  3689.         B.addApproval('Code-Review', 2)

  3690. 

  3691.         self.executor_server.failJob('job1', A)

  3692. 

  3693.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  3694.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  3695. 

  3696.         self.waitUntilSettled()

  3697. 

  3698.         self.assertBuilds([

  3699.             dict(name='job1', changes='1,1'),

  3700.             dict(name='job2', changes='1,1'),

  3701.             dict(name='job1', changes='1,1 2,1'),

  3702.             dict(name='job2', changes='1,1 2,1'),

  3703.         ])

  3704. 

  3705.         self.release(self.builds[0])

  3706.         self.waitUntilSettled()

  3707. 

  3708.         self.assertBuilds([

  3709.             dict(name='job2', changes='1,1'),

  3710.             dict(name='job1', changes='2,1'),

  3711.             dict(name='job2', changes='2,1'),

  3712.         ])

  3713. 

  3714.         # Unordered history comparison because the aborts can finish

  3715.         # in any order.

  3716.         self.assertHistory([

  3717.             dict(name='job1', result='FAILURE', changes='1,1'),

  3718.             dict(name='job1', result='ABORTED', changes='1,1 2,1'),

  3719.             dict(name='job2', result='ABORTED', changes='1,1 2,1'),

  3720.         ], ordered=False)

  3721. 

  3722.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3723.         self.waitUntilSettled()

  3724. 

  3725.         self.executor_server.hold_jobs_in_build = False

  3726.         self.executor_server.release()

  3727.         self.waitUntilSettled()

  3728. 

  3729.         self.assertBuilds([])

  3730. 

  3731.         self.assertHistory([

  3732.             dict(name='job1', result='FAILURE', changes='1,1'),

  3733.             dict(name='job1', result='ABORTED', changes='1,1 2,1'),

  3734.             dict(name='job2', result='ABORTED', changes='1,1 2,1'),

  3735.             dict(name='job2', result='SUCCESS', changes='1,1'),

  3736.             dict(name='job1', result='SUCCESS', changes='2,1'),

  3737.             dict(name='job2', result='SUCCESS', changes='2,1'),

  3738.         ], ordered=False)

  3739.         self.assertEqual(A.data['status'], 'NEW')

  3740.         self.assertEqual(B.data['status'], 'MERGED')

  3741.         self.assertEqual(A.reported, 2)

  3742.         self.assertEqual(B.reported, 2)

  3743. 

  3744.     def test_delayed_repo_init(self):

  3745.         self.init_repo("org/new-project")

  3746.         files = {'README': ''}

  3747.         self.addCommitToRepo("org/new-project", 'Initial commit',

  3748.                              files=files, tag='init')

  3749.         self.newTenantConfig('tenants/delayed-repo-init.yaml')

  3750.         self.commitConfigUpdate(

  3751.             'common-config',

  3752.             'layouts/delayed-repo-init.yaml')

  3753.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3754.         self.waitUntilSettled()

  3755. 

  3756.         A = self.fake_gerrit.addFakeChange('org/new-project', 'master', 'A')

  3757. 

  3758.         A.addApproval('Code-Review', 2)

  3759.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  3760.         self.waitUntilSettled()

  3761.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  3762.                          'SUCCESS')

  3763.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  3764.                          'SUCCESS')

  3765.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  3766.                          'SUCCESS')

  3767.         self.assertEqual(A.data['status'], 'MERGED')

  3768.         self.assertEqual(A.reported, 2)

  3769. 

  3770.     @simple_layout('layouts/single-job-with-nodeset.yaml')

  3771.     def test_live_reconfiguration_queued_node_requests(self):

  3772.         # Test that a job with a queued node request still has the

  3773.         # correct state after reconfiguration.

  3774.         self.fake_nodepool.pause()

  3775.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  3776.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  3777.         self.waitUntilSettled()

  3778. 

  3779.         def get_job():

  3780.             data = json.loads(self.scheds.first.sched

  3781.                               .formatStatusJSON('tenant-one'))

  3782.             for pipeline in data['pipelines']:

  3783.                 for queue in pipeline['change_queues']:

  3784.                     for head in queue['heads']:

  3785.                         for item in head:

  3786.                             for job in item['jobs']:

  3787.                                 if job['name'] == 'check-job':

  3788.                                     return job

  3789. 

  3790.         job = get_job()

  3791.         self.assertTrue(job['queued'])

  3792. 

  3793.         self.scheds.execute(lambda app: app.sched.reconfigure(self.config))

  3794.         self.waitUntilSettled()

  3795. 

  3796.         job = get_job()

  3797.         self.assertTrue(job['queued'])

  3798. 

  3799.         self.fake_nodepool.unpause()

  3800.         self.waitUntilSettled()

  3801.         self.assertHistory([

  3802.             dict(name='check-job', result='SUCCESS', changes='1,1'),

  3803.         ])

  3804. 

  3805.     @simple_layout('layouts/repo-deleted.yaml')

  3806.     def test_repo_deleted(self):

  3807.         self.init_repo("org/delete-project")

  3808.         A = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'A')

  3809. 

  3810.         A.addApproval('Code-Review', 2)

  3811.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  3812.         self.waitUntilSettled()

  3813.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  3814.                          'SUCCESS')

  3815.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  3816.                          'SUCCESS')

  3817.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  3818.                          'SUCCESS')

  3819.         self.assertEqual(A.data['status'], 'MERGED')

  3820.         self.assertEqual(A.reported, 2)

  3821. 

  3822.         # Delete org/new-project zuul repo. Should be recloned.

  3823.         p = 'org/delete-project'

  3824.         if os.path.exists(os.path.join(self.merger_src_root, p)):

  3825.             shutil.rmtree(os.path.join(self.merger_src_root, p))

  3826.         if os.path.exists(os.path.join(self.executor_src_root, p)):

  3827.             shutil.rmtree(os.path.join(self.executor_src_root, p))

  3828. 

  3829.         B = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'B')

  3830. 

  3831.         B.addApproval('Code-Review', 2)

  3832.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  3833.         self.waitUntilSettled()

  3834.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  3835.                          'SUCCESS')

  3836.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  3837.                          'SUCCESS')

  3838.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  3839.                          'SUCCESS')

  3840.         self.assertEqual(B.data['status'], 'MERGED')

  3841.         self.assertEqual(B.reported, 2)

  3842. 

  3843.     @simple_layout('layouts/untrusted-secrets.yaml')

  3844.     def test_untrusted_secrets(self):

  3845.         "Test untrusted secrets"

  3846.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  3847.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  3848.         self.waitUntilSettled()

  3849. 

  3850.         self.assertHistory([])

  3851.         self.assertEqual(A.patchsets[0]['approvals'][0]['value'], "-1")

  3852.         self.assertIn('does not allow post-review job',

  3853.                       A.messages[0])

  3854. 

  3855.     @simple_layout('layouts/tags.yaml')

  3856.     def test_tags(self):

  3857.         "Test job tags"

  3858.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  3859.         B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')

  3860.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  3861.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  3862.         self.waitUntilSettled()

  3863. 

  3864.         self.assertEqual(len(self.history), 2)

  3865. 

  3866.         results = {self.getJobFromHistory('merge',

  3867.                    project='org/project1').uuid: ['extratag', 'merge'],

  3868.                    self.getJobFromHistory('merge',

  3869.                    project='org/project2').uuid: ['merge']}

  3870. 

  3871.         for build in self.history:

  3872.             self.assertEqual(results.get(build.uuid, ''),

  3873.                              build.parameters['zuul'].get('jobtags'))

  3874. 

  3875.     def test_timer_template(self):

  3876.         "Test that a periodic job is triggered"

  3877.         # This test can not use simple_layout because it must start

  3878.         # with a configuration which does not include a

  3879.         # timer-triggered job so that we have an opportunity to set

  3880.         # the hold flag before the first job.

  3881.         self.create_branch('org/project', 'stable')

  3882.         self.executor_server.hold_jobs_in_build = True

  3883.         self.commitConfigUpdate('common-config', 'layouts/timer-template.yaml')

  3884.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3885. 

  3886.         # The pipeline triggers every second, so we should have seen

  3887.         # several by now.

  3888.         time.sleep(5)

  3889.         self.waitUntilSettled()

  3890. 

  3891.         self.assertEqual(len(self.builds), 2)

  3892. 

  3893.         merge_count_project1 = 0

  3894.         for job in self.gearman_server.jobs_history:

  3895.             if job.name == b'merger:refstate':

  3896.                 args = job.arguments

  3897.                 if isinstance(args, bytes):

  3898.                     args = args.decode('utf-8')

  3899.                 args = json.loads(args)

  3900.                 if args["items"][0]["project"] == "org/project1":

  3901.                     merge_count_project1 += 1

  3902.         self.assertEquals(merge_count_project1, 0,

  3903.                           "project1 shouldn't have any refstate call")

  3904. 

  3905.         self.executor_server.hold_jobs_in_build = False

  3906.         # Stop queuing timer triggered jobs so that the assertions

  3907.         # below don't race against more jobs being queued.

  3908.         self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')

  3909.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3910.         self.waitUntilSettled()

  3911.         # If APScheduler is in mid-event when we remove the job, we

  3912.         # can end up with one more event firing, so give it an extra

  3913.         # second to settle.

  3914.         time.sleep(1)

  3915.         self.waitUntilSettled()

  3916.         self.executor_server.release()

  3917.         self.waitUntilSettled()

  3918. 

  3919.         self.assertHistory([

  3920.             dict(name='project-bitrot', result='SUCCESS',

  3921.                  ref='refs/heads/master'),

  3922.             dict(name='project-bitrot', result='SUCCESS',

  3923.                  ref='refs/heads/stable'),

  3924.         ], ordered=False)

  3925. 

  3926.     def _test_timer(self, config_file):

  3927.         # This test can not use simple_layout because it must start

  3928.         # with a configuration which does not include a

  3929.         # timer-triggered job so that we have an opportunity to set

  3930.         # the hold flag before the first job.

  3931.         self.create_branch('org/project', 'stable')

  3932.         self.executor_server.hold_jobs_in_build = True

  3933.         self.commitConfigUpdate('common-config', config_file)

  3934.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3935. 

  3936.         # The pipeline triggers every second, so we should have seen

  3937.         # several by now.

  3938.         for _ in iterate_timeout(60, 'jobs started'):

  3939.             if len(self.builds) > 1:

  3940.                 break

  3941. 

  3942.         merge_count_project1 = 0

  3943.         for job in self.gearman_server.jobs_history:

  3944.             if job.name == b'merger:refstate':

  3945.                 args = job.arguments

  3946.                 if isinstance(args, bytes):

  3947.                     args = args.decode('utf-8')

  3948.                 args = json.loads(args)

  3949.                 if args["items"][0]["project"] == "org/project1":

  3950.                     merge_count_project1 += 1

  3951.         self.assertEquals(merge_count_project1, 0,

  3952.                           "project1 shouldn't have any refstate call")

  3953. 

  3954.         # Ensure that the status json has the ref so we can render it in the

  3955.         # web ui.

  3956.         data = json.loads(self.scheds.first.sched

  3957.                           .formatStatusJSON('tenant-one'))

  3958.         pipeline = [x for x in data['pipelines'] if x['name'] == 'periodic'][0]

  3959.         first = pipeline['change_queues'][0]['heads'][0][0]

  3960.         second = pipeline['change_queues'][1]['heads'][0][0]

  3961.         self.assertIn(first['ref'], ['refs/heads/master', 'refs/heads/stable'])

  3962.         self.assertIn(second['ref'],

  3963.                       ['refs/heads/master', 'refs/heads/stable'])

  3964. 

  3965.         self.executor_server.hold_jobs_in_build = False

  3966. 

  3967.         # Stop queuing timer triggered jobs so that the assertions

  3968.         # below don't race against more jobs being queued.

  3969.         self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')

  3970.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  3971.         self.waitUntilSettled()

  3972.         # If APScheduler is in mid-event when we remove the job, we

  3973.         # can end up with one more event firing, so give it an extra

  3974.         # second to settle.

  3975.         time.sleep(3)

  3976.         self.waitUntilSettled()

  3977. 

  3978.         self.executor_server.release()

  3979.         self.waitUntilSettled()

  3980. 

  3981.         self.assertTrue(len(self.history) > 1)

  3982.         for job in self.history[:1]:

  3983.             self.assertEqual(job.result, 'SUCCESS')

  3984.             self.assertEqual(job.name, 'project-bitrot')

  3985.             self.assertIn(job.ref, ('refs/heads/stable', 'refs/heads/master'))

  3986. 

  3987.     def test_timer(self):

  3988.         "Test that a periodic job is triggered"

  3989.         self._test_timer('layouts/timer.yaml')

  3990. 

  3991.     def test_timer_with_jitter(self):

  3992.         "Test that a periodic job with a jitter is triggered"

  3993.         self._test_timer('layouts/timer-jitter.yaml')

  3994. 

  3995.     def test_idle(self):

  3996.         "Test that frequent periodic jobs work"

  3997.         # This test can not use simple_layout because it must start

  3998.         # with a configuration which does not include a

  3999.         # timer-triggered job so that we have an opportunity to set

  4000.         # the hold flag before the first job.

  4001.         self.executor_server.hold_jobs_in_build = True

  4002. 

  4003.         for x in range(1, 3):

  4004.             # Test that timer triggers periodic jobs even across

  4005.             # layout config reloads.

  4006.             # Start timer trigger

  4007.             self.commitConfigUpdate('common-config',

  4008.                                     'layouts/idle.yaml')

  4009.             self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  4010.             self.waitUntilSettled()

  4011. 

  4012.             # The pipeline triggers every second, so we should have seen

  4013.             # several by now.

  4014.             time.sleep(5)

  4015. 

  4016.             # Stop queuing timer triggered jobs so that the assertions

  4017.             # below don't race against more jobs being queued.

  4018.             self.commitConfigUpdate('common-config',

  4019.                                     'layouts/no-timer.yaml')

  4020.             self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  4021.             self.waitUntilSettled()

  4022.             # If APScheduler is in mid-event when we remove the job,

  4023.             # we can end up with one more event firing, so give it an

  4024.             # extra second to settle.

  4025.             time.sleep(1)

  4026.             self.waitUntilSettled()

  4027.             self.assertEqual(len(self.builds), 1,

  4028.                              'Timer builds iteration #%d' % x)

  4029.             self.executor_server.release('.*')

  4030.             self.waitUntilSettled()

  4031.             self.assertEqual(len(self.builds), 0)

  4032.             self.assertEqual(len(self.history), x)

  4033. 

  4034.     @simple_layout('layouts/smtp.yaml')

  4035.     def test_check_smtp_pool(self):

  4036.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4037.         self.waitUntilSettled()

  4038. 

  4039.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  4040.         self.waitUntilSettled()

  4041. 

  4042.         self.assertEqual(len(self.smtp_messages), 2)

  4043. 

  4044.         # A.messages only holds what FakeGerrit places in it. Thus we

  4045.         # work on the knowledge of what the first message should be as

  4046.         # it is only configured to go to SMTP.

  4047. 

  4048.         self.assertEqual('zuul@example.com',

  4049.                          self.smtp_messages[0]['from_email'])

  4050.         self.assertEqual(['you@example.com'],

  4051.                          self.smtp_messages[0]['to_email'])

  4052.         self.assertEqual('Starting check jobs.',

  4053.                          self.smtp_messages[0]['body'])

  4054. 

  4055.         self.assertEqual('zuul_from@example.com',

  4056.                          self.smtp_messages[1]['from_email'])

  4057.         self.assertEqual(['alternative_me@example.com'],

  4058.                          self.smtp_messages[1]['to_email'])

  4059.         self.assertEqual(A.messages[0],

  4060.                          self.smtp_messages[1]['body'])

  4061. 

  4062.     @simple_layout('layouts/smtp.yaml')

  4063.     @mock.patch('zuul.driver.gerrit.gerritreporter.GerritReporter.report')

  4064.     def test_failed_reporter(self, report_mock):

  4065.         '''Test that one failed reporter doesn't break other reporters'''

  4066.         # Warning hacks. We sort the reports here so that the test is

  4067.         # deterministic. Gerrit reporting will fail, but smtp reporting

  4068.         # should succeed.

  4069.         report_mock.side_effect = Exception('Gerrit failed to report')

  4070. 

  4071.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  4072.         check = tenant.layout.pipelines['check']

  4073. 

  4074.         check.success_actions = sorted(check.success_actions,

  4075.                                        key=lambda x: x.name)

  4076.         self.assertEqual(check.success_actions[0].name, 'gerrit')

  4077.         self.assertEqual(check.success_actions[1].name, 'smtp')

  4078. 

  4079.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4080.         self.waitUntilSettled()

  4081.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  4082.         self.waitUntilSettled()

  4083. 

  4084.         # We know that if gerrit ran first and failed and smtp ran second

  4085.         # and sends mail then we handle failures in reporters gracefully.

  4086.         self.assertEqual(len(self.smtp_messages), 2)

  4087. 

  4088.         # A.messages only holds what FakeGerrit places in it. Thus we

  4089.         # work on the knowledge of what the first message should be as

  4090.         # it is only configured to go to SMTP.

  4091. 

  4092.         self.assertEqual('zuul@example.com',

  4093.                          self.smtp_messages[0]['from_email'])

  4094.         self.assertEqual(['you@example.com'],

  4095.                          self.smtp_messages[0]['to_email'])

  4096.         self.assertEqual('Starting check jobs.',

  4097.                          self.smtp_messages[0]['body'])

  4098. 

  4099.         self.assertEqual('zuul_from@example.com',

  4100.                          self.smtp_messages[1]['from_email'])

  4101.         self.assertEqual(['alternative_me@example.com'],

  4102.                          self.smtp_messages[1]['to_email'])

  4103.         # This double checks that Gerrit side failed

  4104.         self.assertEqual(A.messages, [])

  4105. 

  4106.     def test_timer_smtp(self):

  4107.         "Test that a periodic job is triggered"

  4108.         # This test can not use simple_layout because it must start

  4109.         # with a configuration which does not include a

  4110.         # timer-triggered job so that we have an opportunity to set

  4111.         # the hold flag before the first job.

  4112.         self.executor_server.hold_jobs_in_build = True

  4113.         self.commitConfigUpdate('common-config', 'layouts/timer-smtp.yaml')

  4114.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  4115. 

  4116.         # The pipeline triggers every second, so we should have seen

  4117.         # several by now.

  4118.         time.sleep(5)

  4119.         self.waitUntilSettled()

  4120. 

  4121.         self.assertEqual(len(self.builds), 2)

  4122.         self.executor_server.release('.*')

  4123.         self.waitUntilSettled()

  4124.         self.assertEqual(len(self.history), 2)

  4125. 

  4126.         self.assertEqual(self.getJobFromHistory(

  4127.             'project-bitrot-stable-old').result, 'SUCCESS')

  4128.         self.assertEqual(self.getJobFromHistory(

  4129.             'project-bitrot-stable-older').result, 'SUCCESS')

  4130. 

  4131.         self.assertEqual(len(self.smtp_messages), 1)

  4132. 

  4133.         # A.messages only holds what FakeGerrit places in it. Thus we

  4134.         # work on the knowledge of what the first message should be as

  4135.         # it is only configured to go to SMTP.

  4136. 

  4137.         self.assertEqual('zuul_from@example.com',

  4138.                          self.smtp_messages[0]['from_email'])

  4139.         self.assertEqual(['alternative_me@example.com'],

  4140.                          self.smtp_messages[0]['to_email'])

  4141.         self.assertIn('Subject: Periodic check for org/project succeeded',

  4142.                       self.smtp_messages[0]['headers'])

  4143. 

  4144.         # Stop queuing timer triggered jobs and let any that may have

  4145.         # queued through so that end of test assertions pass.

  4146.         self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')

  4147.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  4148.         self.waitUntilSettled()

  4149.         # If APScheduler is in mid-event when we remove the job, we

  4150.         # can end up with one more event firing, so give it an extra

  4151.         # second to settle.

  4152.         time.sleep(1)

  4153.         self.waitUntilSettled()

  4154.         self.executor_server.release('.*')

  4155.         self.waitUntilSettled()

  4156. 

  4157.     @skip("Disabled for early v3 development")

  4158.     def test_timer_sshkey(self):

  4159.         "Test that a periodic job can setup SSH key authentication"

  4160.         self.worker.hold_jobs_in_build = True

  4161.         self.config.set('zuul', 'layout_config',

  4162.                         'tests/fixtures/layout-timer.yaml')

  4163.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  4164.         self.registerJobs()

  4165. 

  4166.         # The pipeline triggers every second, so we should have seen

  4167.         # several by now.

  4168.         time.sleep(5)

  4169.         self.waitUntilSettled()

  4170. 

  4171.         self.assertEqual(len(self.builds), 2)

  4172. 

  4173.         ssh_wrapper = os.path.join(self.git_root, ".ssh_wrapper_gerrit")

  4174.         self.assertTrue(os.path.isfile(ssh_wrapper))

  4175.         with open(ssh_wrapper) as f:

  4176.             ssh_wrapper_content = f.read()

  4177.         self.assertIn("fake_id_rsa", ssh_wrapper_content)

  4178.         # In the unit tests Merger runs in the same process,

  4179.         # so we see its' environment variables

  4180.         self.assertEqual(os.environ['GIT_SSH'], ssh_wrapper)

  4181. 

  4182.         self.worker.release('.*')

  4183.         self.waitUntilSettled()

  4184.         self.assertEqual(len(self.history), 2)

  4185. 

  4186.         self.assertEqual(self.getJobFromHistory(

  4187.             'project-bitrot-stable-old').result, 'SUCCESS')

  4188.         self.assertEqual(self.getJobFromHistory(

  4189.             'project-bitrot-stable-older').result, 'SUCCESS')

  4190. 

  4191.         # Stop queuing timer triggered jobs and let any that may have

  4192.         # queued through so that end of test assertions pass.

  4193.         self.config.set('zuul', 'layout_config',

  4194.                         'tests/fixtures/layout-no-timer.yaml')

  4195.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  4196.         self.registerJobs()

  4197.         self.waitUntilSettled()

  4198.         # If APScheduler is in mid-event when we remove the job, we

  4199.         # can end up with one more event firing, so give it an extra

  4200.         # second to settle.

  4201.         time.sleep(1)

  4202.         self.waitUntilSettled()

  4203.         self.worker.release('.*')

  4204.         self.waitUntilSettled()

  4205. 

  4206.     def test_client_enqueue_change_with_trigger(self):

  4207.         """Test that the RPC client can enqueue a change with the deprecated

  4208.            trigger argument"""

  4209.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4210.         A.addApproval('Code-Review', 2)

  4211.         A.addApproval('Approved', 1)

  4212. 

  4213.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4214.                                           self.gearman_server.port)

  4215.         self.addCleanup(client.shutdown)

  4216.         r = client.enqueue(tenant='tenant-one',

  4217.                            pipeline='gate',

  4218.                            project='org/project',

  4219.                            trigger='gerrit',

  4220.                            change='1,1')

  4221.         self.waitUntilSettled()

  4222.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  4223.                          'SUCCESS')

  4224.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  4225.                          'SUCCESS')

  4226.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  4227.                          'SUCCESS')

  4228.         self.assertEqual(A.data['status'], 'MERGED')

  4229.         self.assertEqual(A.reported, 2)

  4230.         self.assertEqual(r, True)

  4231. 

  4232.     def test_client_enqueue_change_no_trigger(self):

  4233.         """Test that the RPC client can enqueue a change without an explicit

  4234.            trigger"""

  4235.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4236.         A.addApproval('Code-Review', 2)

  4237.         A.addApproval('Approved', 1)

  4238. 

  4239.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4240.                                           self.gearman_server.port)

  4241.         self.addCleanup(client.shutdown)

  4242.         r = client.enqueue(tenant='tenant-one',

  4243.                            pipeline='gate',

  4244.                            project='org/project',

  4245.                            trigger=None,

  4246.                            change='1,1')

  4247.         self.waitUntilSettled()

  4248.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  4249.                          'SUCCESS')

  4250.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  4251.                          'SUCCESS')

  4252.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  4253.                          'SUCCESS')

  4254.         self.assertEqual(A.data['status'], 'MERGED')

  4255.         self.assertEqual(A.reported, 2)

  4256.         self.assertEqual(r, True)

  4257. 

  4258.     @simple_layout('layouts/three-projects.yaml')

  4259.     def test_client_enqueue_change_wrong_project(self):

  4260.         "Test that an enqueue fails if a change doesn't belong to the project"

  4261.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  4262.         A.addApproval('Code-Review', 2)

  4263.         A.addApproval('Approved', 1)

  4264. 

  4265.         B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')

  4266.         B.addApproval('Code-Review', 2)

  4267.         B.addApproval('Approved', 1)

  4268. 

  4269.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4270.                                           self.gearman_server.port)

  4271.         self.addCleanup(client.shutdown)

  4272.         with testtools.ExpectedException(

  4273.             zuul.rpcclient.RPCFailure,

  4274.             'Change 2,1 does not belong to project "org/project1"'):

  4275.             r = client.enqueue(tenant='tenant-one',

  4276.                                pipeline='gate',

  4277.                                project='org/project1',

  4278.                                trigger=None,

  4279.                                change='2,1')

  4280.             self.assertEqual(r, False)

  4281.         self.waitUntilSettled()

  4282. 

  4283.     def test_client_enqueue_ref(self):

  4284.         "Test that the RPC client can enqueue a ref"

  4285.         p = "review.example.com/org/project"

  4286.         upstream = self.getUpstreamRepos([p])

  4287.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4288.         A.setMerged()

  4289.         A_commit = str(upstream[p].commit('master'))

  4290.         self.log.debug("A commit: %s" % A_commit)

  4291. 

  4292.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4293.                                           self.gearman_server.port)

  4294.         self.addCleanup(client.shutdown)

  4295.         r = client.enqueue_ref(

  4296.             tenant='tenant-one',

  4297.             pipeline='post',

  4298.             project='org/project',

  4299.             trigger=None,

  4300.             ref='master',

  4301.             oldrev='90f173846e3af9154517b88543ffbd1691f31366',

  4302.             newrev=A_commit)

  4303.         self.waitUntilSettled()

  4304.         job_names = [x.name for x in self.history]

  4305.         self.assertEqual(len(self.history), 1)

  4306.         self.assertIn('project-post', job_names)

  4307.         self.assertEqual(r, True)

  4308. 

  4309.     def test_client_dequeue_ref(self):

  4310.         "Test that the RPC client can dequeue a ref"

  4311. 

  4312.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4313.                                           self.gearman_server.port)

  4314.         self.addCleanup(client.shutdown)

  4315. 

  4316.         self.executor_server.hold_jobs_in_build = True

  4317. 

  4318.         p = "review.example.com/org/project"

  4319.         upstream = self.getUpstreamRepos([p])

  4320.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4321.         A.setMerged()

  4322.         A_commit = str(upstream[p].commit('master'))

  4323.         self.log.debug("A commit: %s" % A_commit)

  4324. 

  4325.         r = client.enqueue_ref(

  4326.             tenant='tenant-one',

  4327.             pipeline='post',

  4328.             project='org/project',

  4329.             trigger=None,

  4330.             ref='master',

  4331.             oldrev='90f173846e3af9154517b88543ffbd1691f31366',

  4332.             newrev=A_commit)

  4333. 

  4334.         p = "review.example.com/org/project1"

  4335.         upstream = self.getUpstreamRepos([p])

  4336.         B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')

  4337.         B.setMerged()

  4338.         B_commit = str(upstream[p].commit('master'))

  4339.         self.log.debug("B commit: %s" % B_commit)

  4340. 

  4341.         r = client.enqueue_ref(

  4342.             tenant='tenant-one',

  4343.             pipeline='post',

  4344.             project='org/project1',

  4345.             trigger=None,

  4346.             ref='master',

  4347.             oldrev='90f173846e3af9154517b88543ffbd1691f31366',

  4348.             newrev=B_commit)

  4349. 

  4350.         self.waitUntilSettled()

  4351.         r = client.dequeue(

  4352.             tenant='tenant-one',

  4353.             pipeline='post',

  4354.             project='org/project1',

  4355.             change=None,

  4356.             ref='master')

  4357.         self.waitUntilSettled()

  4358.         self.executor_server.release('.*')

  4359.         self.waitUntilSettled()

  4360.         job_names = [x.name for x in self.history]

  4361.         self.assertEqual(len(self.history), 2)

  4362.         aborted_build = [

  4363.             build for build in self.history if

  4364.             build.result == 'ABORTED'][0]

  4365.         self.assertEqual(aborted_build.newrev, B_commit)

  4366.         self.assertIn('project-post', job_names)

  4367.         self.assertEqual(r, True)

  4368. 

  4369.     def test_client_dequeue_dependent_change(self):

  4370.         "Test that the RPC client can dequeue a change"

  4371.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4372.                                           self.gearman_server.port)

  4373.         self.addCleanup(client.shutdown)

  4374. 

  4375.         self.executor_server.hold_jobs_in_build = True

  4376.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4377.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  4378.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  4379. 

  4380.         C.setDependsOn(B, 1)

  4381.         B.setDependsOn(A, 1)

  4382. 

  4383.         A.addApproval('Code-Review', 2)

  4384.         B.addApproval('Code-Review', 2)

  4385.         C.addApproval('Code-Review', 2)

  4386. 

  4387.         # Promote to 'gate' pipeline

  4388.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  4389.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  4390.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  4391.         self.waitUntilSettled()

  4392. 

  4393.         client.dequeue(

  4394.             tenant='tenant-one',

  4395.             pipeline='gate',

  4396.             project='org/project',

  4397.             change='1,1',

  4398.             ref=None)

  4399. 

  4400.         self.waitUntilSettled()

  4401. 

  4402.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  4403.         gate_pipeline = tenant.layout.pipelines['gate']

  4404.         self.assertEqual(gate_pipeline.getAllItems(), [])

  4405.         self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1)

  4406. 

  4407.         self.executor_server.hold_jobs_in_build = False

  4408.         self.executor_server.release()

  4409.         self.waitUntilSettled()

  4410. 

  4411.     def test_client_dequeue_independent_change(self):

  4412.         "Test that the RPC client can dequeue a change"

  4413. 

  4414.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4415.                                           self.gearman_server.port)

  4416.         self.addCleanup(client.shutdown)

  4417. 

  4418.         self.executor_server.hold_jobs_in_build = True

  4419.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4420.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  4421.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  4422. 

  4423.         A.addApproval('Code-Review', 2)

  4424.         B.addApproval('Code-Review', 2)

  4425.         C.addApproval('Code-Review', 2)

  4426. 

  4427.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  4428.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  4429.         self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

  4430.         self.waitUntilSettled()

  4431. 

  4432.         client.dequeue(

  4433.             tenant='tenant-one',

  4434.             pipeline='check',

  4435.             project='org/project',

  4436.             change='1,1',

  4437.             ref=None)

  4438.         self.waitUntilSettled()

  4439. 

  4440.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  4441.         check_pipeline = tenant.layout.pipelines['check']

  4442.         self.assertEqual(len(check_pipeline.getAllItems()), 2)

  4443.         self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1)

  4444. 

  4445.         self.executor_server.hold_jobs_in_build = False

  4446.         self.executor_server.release()

  4447.         self.waitUntilSettled()

  4448. 

  4449.     @simple_layout('layouts/three-projects.yaml')

  4450.     def test_client_dequeue_wrong_project(self):

  4451.         "Test that dequeue fails if change and project do not match"

  4452. 

  4453.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4454.                                           self.gearman_server.port)

  4455.         self.addCleanup(client.shutdown)

  4456. 

  4457.         self.executor_server.hold_jobs_in_build = True

  4458.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  4459.         B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')

  4460.         C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')

  4461.         D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')

  4462. 

  4463.         A.addApproval('Code-Review', 2)

  4464.         B.addApproval('Code-Review', 2)

  4465.         C.addApproval('Code-Review', 2)

  4466.         D.addApproval('Code-Review', 2)

  4467. 

  4468.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  4469.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  4470.         self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

  4471.         self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))

  4472.         self.waitUntilSettled()

  4473. 

  4474.         with testtools.ExpectedException(

  4475.             zuul.rpcclient.RPCFailure,

  4476.             'Change 4,1 does not belong to project "org/project1"'):

  4477.             r = client.dequeue(

  4478.                 tenant='tenant-one',

  4479.                 pipeline='check',

  4480.                 project='org/project1',

  4481.                 change='4,1',

  4482.                 ref=None)

  4483.             self.waitUntilSettled()

  4484.             self.assertEqual(r, False)

  4485. 

  4486.         self.executor_server.hold_jobs_in_build = False

  4487.         self.executor_server.release()

  4488.         self.waitUntilSettled()

  4489. 

  4490.     def test_client_dequeue_change_by_ref(self):

  4491.         "Test that the RPC client can dequeue a change by ref"

  4492.         # Test this on the periodic pipeline, where it makes most sense to

  4493.         # use ref

  4494.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4495.                                           self.gearman_server.port)

  4496.         self.addCleanup(client.shutdown)

  4497. 

  4498.         self.create_branch('org/project', 'stable')

  4499.         self.fake_gerrit.addEvent(

  4500.             self.fake_gerrit.getFakeBranchCreatedEvent(

  4501.                 'org/project', 'stable'))

  4502.         self.waitUntilSettled()

  4503.         self.executor_server.hold_jobs_in_build = True

  4504.         self.commitConfigUpdate('common-config', 'layouts/timer.yaml')

  4505.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  4506. 

  4507.         # We expect that one build for each branch (master and stable) appears.

  4508.         for _ in iterate_timeout(30, 'Wait for two builds that are hold'):

  4509.             if len(self.builds) == 2:

  4510.                 break

  4511.         self.waitUntilSettled()

  4512. 

  4513.         client.dequeue(

  4514.             tenant='tenant-one',

  4515.             pipeline='periodic',

  4516.             project='org/project',

  4517.             change=None,

  4518.             ref='refs/heads/stable')

  4519.         self.waitUntilSettled()

  4520. 

  4521.         self.commitConfigUpdate('common-config',

  4522.                                 'layouts/no-timer.yaml')

  4523.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  4524.         self.waitUntilSettled()

  4525.         self.executor_server.hold_jobs_in_build = False

  4526.         self.executor_server.release()

  4527.         self.waitUntilSettled()

  4528.         self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1)

  4529. 

  4530.     def test_client_enqueue_negative(self):

  4531.         "Test that the RPC client returns errors"

  4532.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4533.                                           self.gearman_server.port)

  4534.         self.addCleanup(client.shutdown)

  4535.         with testtools.ExpectedException(zuul.rpcclient.RPCFailure,

  4536.                                          "Invalid tenant"):

  4537.             r = client.enqueue(tenant='tenant-foo',

  4538.                                pipeline='gate',

  4539.                                project='org/project',

  4540.                                trigger=None,

  4541.                                change='1,1')

  4542.             self.assertEqual(r, False)

  4543. 

  4544.         with testtools.ExpectedException(zuul.rpcclient.RPCFailure,

  4545.                                          "Invalid project"):

  4546.             r = client.enqueue(tenant='tenant-one',

  4547.                                pipeline='gate',

  4548.                                project='project-does-not-exist',

  4549.                                trigger=None,

  4550.                                change='1,1')

  4551.             self.assertEqual(r, False)

  4552. 

  4553.         with testtools.ExpectedException(zuul.rpcclient.RPCFailure,

  4554.                                          "Invalid pipeline"):

  4555.             r = client.enqueue(tenant='tenant-one',

  4556.                                pipeline='pipeline-does-not-exist',

  4557.                                project='org/project',

  4558.                                trigger=None,

  4559.                                change='1,1')

  4560.             self.assertEqual(r, False)

  4561. 

  4562.         with testtools.ExpectedException(zuul.rpcclient.RPCFailure,

  4563.                                          "Invalid change"):

  4564.             r = client.enqueue(tenant='tenant-one',

  4565.                                pipeline='gate',

  4566.                                project='org/project',

  4567.                                trigger=None,

  4568.                                change='1,1')

  4569.             self.assertEqual(r, False)

  4570. 

  4571.         self.waitUntilSettled()

  4572.         self.assertEqual(len(self.history), 0)

  4573.         self.assertEqual(len(self.builds), 0)

  4574. 

  4575.     def test_client_enqueue_ref_negative(self):

  4576.         "Test that the RPC client returns errors"

  4577.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4578.                                           self.gearman_server.port)

  4579.         self.addCleanup(client.shutdown)

  4580.         with testtools.ExpectedException(zuul.rpcclient.RPCFailure,

  4581.                                          "New rev must be 40 character sha1"):

  4582.             r = client.enqueue_ref(

  4583.                 tenant='tenant-one',

  4584.                 pipeline='post',

  4585.                 project='org/project',

  4586.                 trigger=None,

  4587.                 ref='master',

  4588.                 oldrev='90f173846e3af9154517b88543ffbd1691f31366',

  4589.                 newrev='10054041')

  4590.             self.assertEqual(r, False)

  4591.         with testtools.ExpectedException(zuul.rpcclient.RPCFailure,

  4592.                                          "Old rev must be 40 character sha1"):

  4593.             r = client.enqueue_ref(

  4594.                 tenant='tenant-one',

  4595.                 pipeline='post',

  4596.                 project='org/project',

  4597.                 trigger=None,

  4598.                 ref='master',

  4599.                 oldrev='10054041',

  4600.                 newrev='90f173846e3af9154517b88543ffbd1691f31366')

  4601.             self.assertEqual(r, False)

  4602.         with testtools.ExpectedException(zuul.rpcclient.RPCFailure,

  4603.                                          "New rev must be base16 hash"):

  4604.             r = client.enqueue_ref(

  4605.                 tenant='tenant-one',

  4606.                 pipeline='post',

  4607.                 project='org/project',

  4608.                 trigger=None,

  4609.                 ref='master',

  4610.                 oldrev='90f173846e3af9154517b88543ffbd1691f31366',

  4611.                 newrev='notbase16')

  4612.             self.assertEqual(r, False)

  4613.         with testtools.ExpectedException(zuul.rpcclient.RPCFailure,

  4614.                                          "Old rev must be base16 hash"):

  4615.             r = client.enqueue_ref(

  4616.                 tenant='tenant-one',

  4617.                 pipeline='post',

  4618.                 project='org/project',

  4619.                 trigger=None,

  4620.                 ref='master',

  4621.                 oldrev='notbase16',

  4622.                 newrev='90f173846e3af9154517b88543ffbd1691f31366')

  4623.             self.assertEqual(r, False)

  4624. 

  4625.     def test_client_promote(self):

  4626.         "Test that the RPC client can promote a change"

  4627.         self.executor_server.hold_jobs_in_build = True

  4628.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4629.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  4630.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  4631.         A.addApproval('Code-Review', 2)

  4632.         B.addApproval('Code-Review', 2)

  4633.         C.addApproval('Code-Review', 2)

  4634. 

  4635.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  4636.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  4637.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  4638. 

  4639.         self.waitUntilSettled()

  4640. 

  4641.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  4642.         items = tenant.layout.pipelines['gate'].getAllItems()

  4643.         enqueue_times = {}

  4644.         for item in items:

  4645.             enqueue_times[str(item.change)] = item.enqueue_time

  4646. 

  4647.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4648.                                           self.gearman_server.port)

  4649.         self.addCleanup(client.shutdown)

  4650.         r = client.promote(tenant='tenant-one',

  4651.                            pipeline='gate',

  4652.                            change_ids=['2,1', '3,1'])

  4653. 

  4654.         # ensure that enqueue times are durable

  4655.         items = tenant.layout.pipelines['gate'].getAllItems()

  4656.         for item in items:

  4657.             self.assertEqual(

  4658.                 enqueue_times[str(item.change)], item.enqueue_time)

  4659. 

  4660.         self.waitUntilSettled()

  4661.         self.executor_server.release('.*-merge')

  4662.         self.waitUntilSettled()

  4663.         self.executor_server.release('.*-merge')

  4664.         self.waitUntilSettled()

  4665.         self.executor_server.release('.*-merge')

  4666.         self.waitUntilSettled()

  4667. 

  4668.         self.assertEqual(len(self.builds), 6)

  4669.         self.assertEqual(self.builds[0].name, 'project-test1')

  4670.         self.assertEqual(self.builds[1].name, 'project-test2')

  4671.         self.assertEqual(self.builds[2].name, 'project-test1')

  4672.         self.assertEqual(self.builds[3].name, 'project-test2')

  4673.         self.assertEqual(self.builds[4].name, 'project-test1')

  4674.         self.assertEqual(self.builds[5].name, 'project-test2')

  4675. 

  4676.         self.assertTrue(self.builds[0].hasChanges(B))

  4677.         self.assertFalse(self.builds[0].hasChanges(A))

  4678.         self.assertFalse(self.builds[0].hasChanges(C))

  4679. 

  4680.         self.assertTrue(self.builds[2].hasChanges(B))

  4681.         self.assertTrue(self.builds[2].hasChanges(C))

  4682.         self.assertFalse(self.builds[2].hasChanges(A))

  4683. 

  4684.         self.assertTrue(self.builds[4].hasChanges(B))

  4685.         self.assertTrue(self.builds[4].hasChanges(C))

  4686.         self.assertTrue(self.builds[4].hasChanges(A))

  4687. 

  4688.         self.executor_server.release()

  4689.         self.waitUntilSettled()

  4690. 

  4691.         self.assertEqual(A.data['status'], 'MERGED')

  4692.         self.assertEqual(A.reported, 2)

  4693.         self.assertEqual(B.data['status'], 'MERGED')

  4694.         self.assertEqual(B.reported, 2)

  4695.         self.assertEqual(C.data['status'], 'MERGED')

  4696.         self.assertEqual(C.reported, 2)

  4697. 

  4698.         self.assertEqual(r, True)

  4699. 

  4700.     def test_client_promote_dependent(self):

  4701.         "Test that the RPC client can promote a dependent change"

  4702.         # C (depends on B) -> B -> A ; then promote C to get:

  4703.         # A -> C (depends on B) -> B

  4704.         self.executor_server.hold_jobs_in_build = True

  4705.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4706.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  4707.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  4708. 

  4709.         C.setDependsOn(B, 1)

  4710. 

  4711.         A.addApproval('Code-Review', 2)

  4712.         B.addApproval('Code-Review', 2)

  4713.         C.addApproval('Code-Review', 2)

  4714. 

  4715.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  4716.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  4717.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  4718. 

  4719.         self.waitUntilSettled()

  4720. 

  4721.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4722.                                           self.gearman_server.port)

  4723.         self.addCleanup(client.shutdown)

  4724.         r = client.promote(tenant='tenant-one',

  4725.                            pipeline='gate',

  4726.                            change_ids=['3,1'])

  4727. 

  4728.         self.waitUntilSettled()

  4729.         self.executor_server.release('.*-merge')

  4730.         self.waitUntilSettled()

  4731.         self.executor_server.release('.*-merge')

  4732.         self.waitUntilSettled()

  4733.         self.executor_server.release('.*-merge')

  4734.         self.waitUntilSettled()

  4735. 

  4736.         self.assertEqual(len(self.builds), 6)

  4737.         self.assertEqual(self.builds[0].name, 'project-test1')

  4738.         self.assertEqual(self.builds[1].name, 'project-test2')

  4739.         self.assertEqual(self.builds[2].name, 'project-test1')

  4740.         self.assertEqual(self.builds[3].name, 'project-test2')

  4741.         self.assertEqual(self.builds[4].name, 'project-test1')

  4742.         self.assertEqual(self.builds[5].name, 'project-test2')

  4743. 

  4744.         self.assertTrue(self.builds[0].hasChanges(B))

  4745.         self.assertFalse(self.builds[0].hasChanges(A))

  4746.         self.assertFalse(self.builds[0].hasChanges(C))

  4747. 

  4748.         self.assertTrue(self.builds[2].hasChanges(B))

  4749.         self.assertTrue(self.builds[2].hasChanges(C))

  4750.         self.assertFalse(self.builds[2].hasChanges(A))

  4751. 

  4752.         self.assertTrue(self.builds[4].hasChanges(B))

  4753.         self.assertTrue(self.builds[4].hasChanges(C))

  4754.         self.assertTrue(self.builds[4].hasChanges(A))

  4755. 

  4756.         self.executor_server.release()

  4757.         self.waitUntilSettled()

  4758. 

  4759.         self.assertEqual(A.data['status'], 'MERGED')

  4760.         self.assertEqual(A.reported, 2)

  4761.         self.assertEqual(B.data['status'], 'MERGED')

  4762.         self.assertEqual(B.reported, 2)

  4763.         self.assertEqual(C.data['status'], 'MERGED')

  4764.         self.assertEqual(C.reported, 2)

  4765. 

  4766.         self.assertEqual(r, True)

  4767. 

  4768.     def test_client_promote_negative(self):

  4769.         "Test that the RPC client returns errors for promotion"

  4770.         self.executor_server.hold_jobs_in_build = True

  4771.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4772.         A.addApproval('Code-Review', 2)

  4773.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  4774.         self.waitUntilSettled()

  4775. 

  4776.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  4777.                                           self.gearman_server.port)

  4778.         self.addCleanup(client.shutdown)

  4779. 

  4780.         with testtools.ExpectedException(zuul.rpcclient.RPCFailure):

  4781.             r = client.promote(tenant='tenant-one',

  4782.                                pipeline='nonexistent',

  4783.                                change_ids=['2,1', '3,1'])

  4784.             self.assertEqual(r, False)

  4785. 

  4786.         with testtools.ExpectedException(zuul.rpcclient.RPCFailure):

  4787.             r = client.promote(tenant='tenant-one',

  4788.                                pipeline='gate',

  4789.                                change_ids=['4,1'])

  4790.             self.assertEqual(r, False)

  4791. 

  4792.         self.executor_server.hold_jobs_in_build = False

  4793.         self.executor_server.release()

  4794.         self.waitUntilSettled()

  4795. 

  4796.     @simple_layout('layouts/rate-limit.yaml')

  4797.     def test_queue_rate_limiting(self):

  4798.         "Test that DependentPipelines are rate limited with dep across window"

  4799.         self.executor_server.hold_jobs_in_build = True

  4800.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4801.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  4802.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  4803. 

  4804.         C.setDependsOn(B, 1)

  4805.         self.executor_server.failJob('project-test1', A)

  4806. 

  4807.         A.addApproval('Code-Review', 2)

  4808.         B.addApproval('Code-Review', 2)

  4809.         C.addApproval('Code-Review', 2)

  4810. 

  4811.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  4812.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  4813.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  4814.         self.waitUntilSettled()

  4815. 

  4816.         # Only A and B will have their merge jobs queued because

  4817.         # window is 2.

  4818.         self.assertEqual(len(self.builds), 2)

  4819.         self.assertEqual(self.builds[0].name, 'project-merge')

  4820.         self.assertEqual(self.builds[1].name, 'project-merge')

  4821. 

  4822.         # Release the merge jobs one at a time.

  4823.         self.builds[0].release()

  4824.         self.waitUntilSettled()

  4825.         self.builds[0].release()

  4826.         self.waitUntilSettled()

  4827. 

  4828.         # Only A and B will have their test jobs queued because

  4829.         # window is 2.

  4830.         self.assertEqual(len(self.builds), 4)

  4831.         self.assertEqual(self.builds[0].name, 'project-test1')

  4832.         self.assertEqual(self.builds[1].name, 'project-test2')

  4833.         self.assertEqual(self.builds[2].name, 'project-test1')

  4834.         self.assertEqual(self.builds[3].name, 'project-test2')

  4835. 

  4836.         self.executor_server.release('project-.*')

  4837.         self.waitUntilSettled()

  4838. 

  4839.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  4840.         queue = tenant.layout.pipelines['gate'].queues[0]

  4841.         # A failed so window is reduced by 1 to 1.

  4842.         self.assertEqual(queue.window, 1)

  4843.         self.assertEqual(queue.window_floor, 1)

  4844.         self.assertEqual(A.data['status'], 'NEW')

  4845. 

  4846.         # Gate is reset and only B's merge job is queued because

  4847.         # window shrunk to 1.

  4848.         self.assertEqual(len(self.builds), 1)

  4849.         self.assertEqual(self.builds[0].name, 'project-merge')

  4850. 

  4851.         self.executor_server.release('.*-merge')

  4852.         self.waitUntilSettled()

  4853. 

  4854.         # Only B's test jobs are queued because window is still 1.

  4855.         self.assertEqual(len(self.builds), 2)

  4856.         self.assertEqual(self.builds[0].name, 'project-test1')

  4857.         self.assertEqual(self.builds[1].name, 'project-test2')

  4858. 

  4859.         self.executor_server.release('project-.*')

  4860.         self.waitUntilSettled()

  4861. 

  4862.         # B was successfully merged so window is increased to 2.

  4863.         self.assertEqual(queue.window, 2)

  4864.         self.assertEqual(queue.window_floor, 1)

  4865.         self.assertEqual(B.data['status'], 'MERGED')

  4866. 

  4867.         # Only C is left and its merge job is queued.

  4868.         self.assertEqual(len(self.builds), 1)

  4869.         self.assertEqual(self.builds[0].name, 'project-merge')

  4870. 

  4871.         self.executor_server.release('.*-merge')

  4872.         self.waitUntilSettled()

  4873. 

  4874.         # After successful merge job the test jobs for C are queued.

  4875.         self.assertEqual(len(self.builds), 2)

  4876.         self.assertEqual(self.builds[0].name, 'project-test1')

  4877.         self.assertEqual(self.builds[1].name, 'project-test2')

  4878. 

  4879.         self.executor_server.release('project-.*')

  4880.         self.waitUntilSettled()

  4881. 

  4882.         # C successfully merged so window is bumped to 3.

  4883.         self.assertEqual(queue.window, 3)

  4884.         self.assertEqual(queue.window_floor, 1)

  4885.         self.assertEqual(C.data['status'], 'MERGED')

  4886. 

  4887.     @simple_layout('layouts/rate-limit.yaml')

  4888.     def test_queue_rate_limiting_dependent(self):

  4889.         "Test that DependentPipelines are rate limited with dep in window"

  4890.         self.executor_server.hold_jobs_in_build = True

  4891.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4892.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  4893.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  4894. 

  4895.         B.setDependsOn(A, 1)

  4896. 

  4897.         self.executor_server.failJob('project-test1', A)

  4898. 

  4899.         A.addApproval('Code-Review', 2)

  4900.         B.addApproval('Code-Review', 2)

  4901.         C.addApproval('Code-Review', 2)

  4902. 

  4903.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  4904.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  4905.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  4906.         self.waitUntilSettled()

  4907. 

  4908.         # Only A and B will have their merge jobs queued because

  4909.         # window is 2.

  4910.         self.assertEqual(len(self.builds), 2)

  4911.         self.assertEqual(self.builds[0].name, 'project-merge')

  4912.         self.assertEqual(self.builds[1].name, 'project-merge')

  4913. 

  4914.         self.orderedRelease(2)

  4915. 

  4916.         # Only A and B will have their test jobs queued because

  4917.         # window is 2.

  4918.         self.assertEqual(len(self.builds), 4)

  4919.         self.assertEqual(self.builds[0].name, 'project-test1')

  4920.         self.assertEqual(self.builds[1].name, 'project-test2')

  4921.         self.assertEqual(self.builds[2].name, 'project-test1')

  4922.         self.assertEqual(self.builds[3].name, 'project-test2')

  4923. 

  4924.         self.executor_server.release('project-.*')

  4925.         self.waitUntilSettled()

  4926. 

  4927.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  4928.         queue = tenant.layout.pipelines['gate'].queues[0]

  4929.         # A failed so window is reduced by 1 to 1.

  4930.         self.assertEqual(queue.window, 1)

  4931.         self.assertEqual(queue.window_floor, 1)

  4932.         self.assertEqual(A.data['status'], 'NEW')

  4933.         self.assertEqual(B.data['status'], 'NEW')

  4934. 

  4935.         # Gate is reset and only C's merge job is queued because

  4936.         # window shrunk to 1 and A and B were dequeued.

  4937.         self.assertEqual(len(self.builds), 1)

  4938.         self.assertEqual(self.builds[0].name, 'project-merge')

  4939. 

  4940.         self.orderedRelease(1)

  4941. 

  4942.         # Only C's test jobs are queued because window is still 1.

  4943.         self.assertEqual(len(self.builds), 2)

  4944.         builds = self.getSortedBuilds()

  4945.         self.assertEqual(builds[0].name, 'project-test1')

  4946.         self.assertEqual(builds[1].name, 'project-test2')

  4947. 

  4948.         self.executor_server.release('project-.*')

  4949.         self.waitUntilSettled()

  4950. 

  4951.         # C was successfully merged so window is increased to 2.

  4952.         self.assertEqual(queue.window, 2)

  4953.         self.assertEqual(queue.window_floor, 1)

  4954.         self.assertEqual(C.data['status'], 'MERGED')

  4955. 

  4956.     @simple_layout('layouts/rate-limit-reconfigure.yaml')

  4957.     def test_queue_rate_limiting_reconfigure(self):

  4958.         """Test that changes survive a reconfigure when no longer in window.

  4959. 

  4960.         This is a regression tests for a case that lead to an exception during

  4961.         re-enqueue. The exception happened when former active items had already

  4962.         build results but then dropped out of the active window. During

  4963.         re-enqueue the job graph was not re-initialized because the items were

  4964.         no longer active.

  4965.         """

  4966.         self.executor_server.hold_jobs_in_build = True

  4967.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  4968.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  4969.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  4970.         D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')

  4971. 

  4972.         A.addApproval('Code-Review', 2)

  4973.         B.addApproval('Code-Review', 2)

  4974.         C.addApproval('Code-Review', 2)

  4975.         D.addApproval('Code-Review', 2)

  4976. 

  4977.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  4978.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  4979.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  4980.         self.fake_gerrit.addEvent(D.addApproval('Approved', 1))

  4981.         self.waitUntilSettled()

  4982. 

  4983.         self.assertEqual(len(self.builds), 4)

  4984.         self.assertEqual(self.builds[0].name, 'project-merge')

  4985.         self.assertEqual(self.builds[1].name, 'project-merge')

  4986.         self.assertEqual(self.builds[2].name, 'project-merge')

  4987.         self.assertEqual(self.builds[3].name, 'project-merge')

  4988. 

  4989.         self.orderedRelease(4)

  4990. 

  4991.         self.assertEqual(len(self.builds), 8)

  4992.         self.assertEqual(self.builds[0].name, 'project-test1')

  4993.         self.assertEqual(self.builds[1].name, 'project-test2')

  4994.         self.assertEqual(self.builds[2].name, 'project-test1')

  4995.         self.assertEqual(self.builds[3].name, 'project-test2')

  4996.         self.assertEqual(self.builds[4].name, 'project-test1')

  4997.         self.assertEqual(self.builds[5].name, 'project-test2')

  4998.         self.assertEqual(self.builds[6].name, 'project-test1')

  4999.         self.assertEqual(self.builds[7].name, 'project-test2')

  5000. 

  5001.         self.executor_server.failJob('project-test1', B)

  5002.         self.builds[2].release()

  5003.         self.builds[3].release()

  5004.         self.waitUntilSettled()

  5005. 

  5006.         self.assertEqual(len(self.builds), 4)

  5007.         # A's jobs

  5008.         self.assertEqual(self.builds[0].name, 'project-test1')

  5009.         self.assertEqual(self.builds[1].name, 'project-test2')

  5010.         # C's and D's merge jobs

  5011.         self.assertEqual(self.builds[2].name, 'project-merge')

  5012.         self.assertEqual(self.builds[3].name, 'project-merge')

  5013. 

  5014.         # Release merge jobs of C, D after speculative gate reset

  5015.         self.executor_server.release('project-merge')

  5016.         self.waitUntilSettled()

  5017. 

  5018.         self.assertEqual(len(self.builds), 6)

  5019. 

  5020.         # A's jobs

  5021.         self.assertEqual(self.builds[0].name, 'project-test1')

  5022.         self.assertEqual(self.builds[1].name, 'project-test2')

  5023.         # C's + D's jobs

  5024.         self.assertEqual(self.builds[2].name, 'project-test1')

  5025.         self.assertEqual(self.builds[3].name, 'project-test2')

  5026.         self.assertEqual(self.builds[4].name, 'project-test1')

  5027.         self.assertEqual(self.builds[5].name, 'project-test2')

  5028. 

  5029.         # Fail D's job so we have a build results for an item that

  5030.         # is not in the active window after B is reported

  5031.         # (condition that previously lead to an exception)

  5032.         self.executor_server.failJob('project-test1', D)

  5033.         self.builds[4].release()

  5034.         self.waitUntilSettled()

  5035. 

  5036.         # Release A's jobs

  5037.         self.builds[0].release()

  5038.         self.builds[1].release()

  5039.         self.waitUntilSettled()

  5040. 

  5041.         self.assertEqual(len(self.builds), 3)

  5042. 

  5043.         # C's jobs

  5044.         self.assertEqual(self.builds[0].name, 'project-test1')

  5045.         self.assertEqual(self.builds[1].name, 'project-test2')

  5046.         # D's remaining job

  5047.         self.assertEqual(self.builds[2].name, 'project-test2')

  5048. 

  5049.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5050.         queue = tenant.layout.pipelines['gate'].queues[0]

  5051.         self.assertEqual(queue.window, 1)

  5052. 

  5053.         # D dropped out of the window

  5054.         self.assertFalse(queue.queue[-1].active)

  5055. 

  5056.         self.commitConfigUpdate('org/common-config',

  5057.                                 'layouts/rate-limit-reconfigure2.yaml')

  5058.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  5059.         self.waitUntilSettled()

  5060. 

  5061.         # D's remaining job should still be queued

  5062.         self.assertEqual(len(self.builds), 3)

  5063.         self.executor_server.release('project-.*')

  5064.         self.waitUntilSettled()

  5065. 

  5066.     @simple_layout('layouts/reconfigure-window.yaml')

  5067.     def test_reconfigure_window_shrink(self):

  5068.         # Test the active window shrinking during reconfiguration

  5069.         self.executor_server.hold_jobs_in_build = True

  5070. 

  5071.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5072.         A.addApproval('Code-Review', 2)

  5073.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5074.         self.waitUntilSettled()

  5075.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  5076.         B.addApproval('Code-Review', 2)

  5077.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  5078.         self.waitUntilSettled()

  5079. 

  5080.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5081.         queue = tenant.layout.pipelines['gate'].queues[0]

  5082.         self.assertEqual(queue.window, 20)

  5083.         self.assertTrue(len(self.builds), 4)

  5084. 

  5085.         self.executor_server.release('job1')

  5086.         self.waitUntilSettled()

  5087.         self.commitConfigUpdate('org/common-config',

  5088.                                 'layouts/reconfigure-window2.yaml')

  5089.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  5090.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5091.         queue = tenant.layout.pipelines['gate'].queues[0]

  5092.         # Even though we have configured a smaller window, the value

  5093.         # on the existing shared queue should be used.

  5094.         self.assertEqual(queue.window, 20)

  5095.         self.assertTrue(len(self.builds), 4)

  5096. 

  5097.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  5098.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5099.         queue = tenant.layout.pipelines['gate'].queues[0]

  5100.         self.assertEqual(queue.window, 20)

  5101.         self.assertTrue(len(self.builds), 4)

  5102. 

  5103.         self.executor_server.hold_jobs_in_build = False

  5104.         self.executor_server.release()

  5105. 

  5106.         self.waitUntilSettled()

  5107.         self.assertHistory([

  5108.             dict(name='job1', result='SUCCESS', changes='1,1'),

  5109.             dict(name='job1', result='SUCCESS', changes='1,1 2,1'),

  5110.             dict(name='job2', result='SUCCESS', changes='1,1'),

  5111.             dict(name='job2', result='SUCCESS', changes='1,1 2,1'),

  5112.         ], ordered=False)

  5113. 

  5114.     @simple_layout('layouts/reconfigure-window-fixed.yaml')

  5115.     def test_reconfigure_window_fixed(self):

  5116.         # Test the active window shrinking during reconfiguration

  5117.         self.executor_server.hold_jobs_in_build = True

  5118. 

  5119.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5120.         A.addApproval('Code-Review', 2)

  5121.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5122.         self.waitUntilSettled()

  5123.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  5124.         B.addApproval('Code-Review', 2)

  5125.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  5126.         self.waitUntilSettled()

  5127. 

  5128.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5129.         queue = tenant.layout.pipelines['gate'].queues[0]

  5130.         self.assertEqual(queue.window, 2)

  5131.         self.assertEqual(len(self.builds), 4)

  5132. 

  5133.         self.waitUntilSettled()

  5134.         self.commitConfigUpdate('org/common-config',

  5135.                                 'layouts/reconfigure-window-fixed2.yaml')

  5136.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  5137.         self.waitUntilSettled()

  5138.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5139.         queue = tenant.layout.pipelines['gate'].queues[0]

  5140.         # Because we have configured a static window, it should

  5141.         # be allowed to shrink on reconfiguration.

  5142.         self.assertEqual(queue.window, 1)

  5143.         # B is outside the window, but still marked active until the

  5144.         # next pass through the queue processor.

  5145.         self.assertEqual(len(self.builds), 4)

  5146. 

  5147.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  5148.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5149.         queue = tenant.layout.pipelines['gate'].queues[0]

  5150.         self.assertEqual(queue.window, 1)

  5151.         self.waitUntilSettled()

  5152.         # B's builds should not be canceled

  5153.         self.assertEqual(len(self.builds), 4)

  5154. 

  5155.         self.executor_server.hold_jobs_in_build = False

  5156.         self.executor_server.release()

  5157. 

  5158.         self.waitUntilSettled()

  5159.         self.assertHistory([

  5160.             dict(name='job1', result='SUCCESS', changes='1,1'),

  5161.             dict(name='job2', result='SUCCESS', changes='1,1'),

  5162.             dict(name='job1', result='SUCCESS', changes='1,1 2,1'),

  5163.             dict(name='job2', result='SUCCESS', changes='1,1 2,1'),

  5164.         ], ordered=False)

  5165. 

  5166.     @simple_layout('layouts/reconfigure-window-fixed.yaml')

  5167.     def test_reconfigure_window_fixed_requests(self):

  5168.         # Test the active window shrinking during reconfiguration with

  5169.         # outstanding node requests

  5170.         self.executor_server.hold_jobs_in_build = True

  5171. 

  5172.         # Start the jobs for A

  5173.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5174.         A.addApproval('Code-Review', 2)

  5175.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5176.         self.waitUntilSettled()

  5177.         self.log.debug("A complete")

  5178. 

  5179.         # Hold node requests for B

  5180.         self.fake_nodepool.pause()

  5181.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  5182.         B.addApproval('Code-Review', 2)

  5183.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  5184.         self.waitUntilSettled()

  5185.         self.log.debug("B complete")

  5186. 

  5187.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5188.         queue = tenant.layout.pipelines['gate'].queues[0]

  5189.         self.assertEqual(queue.window, 2)

  5190.         self.assertEqual(len(self.builds), 2)

  5191. 

  5192.         self.waitUntilSettled()

  5193.         self.commitConfigUpdate('org/common-config',

  5194.                                 'layouts/reconfigure-window-fixed2.yaml')

  5195.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  5196.         self.waitUntilSettled()

  5197.         self.log.debug("Reconfiguration complete")

  5198. 

  5199.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5200.         queue = tenant.layout.pipelines['gate'].queues[0]

  5201.         # Because we have configured a static window, it should

  5202.         # be allowed to shrink on reconfiguration.

  5203.         self.assertEqual(queue.window, 1)

  5204.         self.assertEqual(len(self.builds), 2)

  5205. 

  5206.         # After the previous reconfig, the queue processor will have

  5207.         # run and marked B inactive; run another reconfiguration so

  5208.         # that we're testing what happens when we reconfigure after

  5209.         # the active window having shrunk.

  5210.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  5211. 

  5212.         # Unpause the node requests now

  5213.         self.fake_nodepool.unpause()

  5214.         self.waitUntilSettled()

  5215.         self.log.debug("Nodepool unpause complete")

  5216. 

  5217.         # Allow A to merge and B to enter the active window and complete

  5218.         self.executor_server.hold_jobs_in_build = False

  5219.         self.executor_server.release()

  5220.         self.waitUntilSettled()

  5221.         self.log.debug("Executor unpause complete")

  5222. 

  5223.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5224.         queue = tenant.layout.pipelines['gate'].queues[0]

  5225.         self.assertEqual(queue.window, 1)

  5226. 

  5227.         self.waitUntilSettled()

  5228.         self.assertHistory([

  5229.             dict(name='job1', result='SUCCESS', changes='1,1'),

  5230.             dict(name='job2', result='SUCCESS', changes='1,1'),

  5231.             dict(name='job1', result='SUCCESS', changes='1,1 2,1'),

  5232.             dict(name='job2', result='SUCCESS', changes='1,1 2,1'),

  5233.         ], ordered=False)

  5234. 

  5235.     @simple_layout('layouts/reconfigure-remove-add.yaml')

  5236.     def test_reconfigure_remove_add(self):

  5237.         # Test removing, then adding a job while in queue

  5238.         self.executor_server.hold_jobs_in_build = True

  5239. 

  5240.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5241.         A.addApproval('Code-Review', 2)

  5242.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5243.         self.waitUntilSettled()

  5244.         self.assertTrue(len(self.builds), 2)

  5245.         self.executor_server.release('job2')

  5246.         self.assertTrue(len(self.builds), 1)

  5247. 

  5248.         # Remove job2

  5249.         self.commitConfigUpdate('org/common-config',

  5250.                                 'layouts/reconfigure-remove-add2.yaml')

  5251.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  5252.         self.assertTrue(len(self.builds), 1)

  5253. 

  5254.         # Add job2 back

  5255.         self.commitConfigUpdate('org/common-config',

  5256.                                 'layouts/reconfigure-remove-add.yaml')

  5257.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  5258.         self.assertTrue(len(self.builds), 2)

  5259. 

  5260.         self.executor_server.hold_jobs_in_build = False

  5261.         self.executor_server.release()

  5262.         # This will run new builds for B

  5263.         self.waitUntilSettled()

  5264.         self.assertHistory([

  5265.             dict(name='job2', result='SUCCESS', changes='1,1'),

  5266.             dict(name='job1', result='SUCCESS', changes='1,1'),

  5267.             dict(name='job2', result='SUCCESS', changes='1,1'),

  5268.         ], ordered=False)

  5269. 

  5270.     def test_worker_update_metadata(self):

  5271.         "Test if a worker can send back metadata about itself"

  5272.         self.executor_server.hold_jobs_in_build = True

  5273. 

  5274.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5275.         A.addApproval('Code-Review', 2)

  5276.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5277.         self.waitUntilSettled()

  5278. 

  5279.         self.assertEqual(len(self.scheds.first.sched.executor.builds), 1)

  5280. 

  5281.         self.log.debug('Current builds:')

  5282.         self.log.debug(self.scheds.first.sched.executor.builds)

  5283. 

  5284.         start = time.time()

  5285.         while True:

  5286.             if time.time() - start > 10:

  5287.                 raise Exception("Timeout waiting for gearman server to report "

  5288.                                 + "back to the client")

  5289.             build = list(self.scheds.first.sched.executor.builds.values())[0]

  5290.             if build.worker.name == self.executor_server.hostname:

  5291.                 break

  5292.             else:

  5293.                 time.sleep(0)

  5294. 

  5295.         self.log.debug(build)

  5296.         self.assertEqual(self.executor_server.hostname, build.worker.name)

  5297. 

  5298.         self.executor_server.hold_jobs_in_build = False

  5299.         self.executor_server.release()

  5300.         self.waitUntilSettled()

  5301. 

  5302.     @simple_layout('layouts/footer-message.yaml')

  5303.     def test_footer_message(self):

  5304.         "Test a pipeline's footer message is correctly added to the report."

  5305.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5306.         A.addApproval('Code-Review', 2)

  5307.         self.executor_server.failJob('project-test1', A)

  5308.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5309.         self.waitUntilSettled()

  5310. 

  5311.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  5312.         B.addApproval('Code-Review', 2)

  5313.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  5314.         self.waitUntilSettled()

  5315. 

  5316.         self.assertEqual(2, len(self.smtp_messages))

  5317. 

  5318.         failure_msg = """\

  5319. Build failed.  For information on how to proceed, see \

  5320. http://wiki.example.org/Test_Failures"""

  5321. 

  5322.         footer_msg = """\

  5323. For CI problems and help debugging, contact ci@example.org"""

  5324. 

  5325.         self.assertTrue(self.smtp_messages[0]['body'].startswith(failure_msg))

  5326.         self.assertTrue(self.smtp_messages[0]['body'].endswith(footer_msg))

  5327.         self.assertFalse(self.smtp_messages[1]['body'].startswith(failure_msg))

  5328.         self.assertTrue(self.smtp_messages[1]['body'].endswith(footer_msg))

  5329. 

  5330.     @simple_layout('layouts/start-message.yaml')

  5331.     def test_start_message(self):

  5332.         "Test a pipeline's start message is correctly added to the report."

  5333.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5334.         A.addApproval('Code-Review', 2)

  5335.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5336.         self.waitUntilSettled()

  5337. 

  5338.         self.assertEqual(1, len(self.smtp_messages))

  5339.         start_msg = "Jobs started in gate for 1,1."

  5340.         self.assertTrue(self.smtp_messages[0]['body'].startswith(start_msg))

  5341. 

  5342.     @simple_layout('layouts/unmanaged-project.yaml')

  5343.     def test_unmanaged_project_start_message(self):

  5344.         "Test start reporting is not done for unmanaged projects."

  5345.         self.init_repo("org/project", tag='init')

  5346.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5347.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  5348.         self.waitUntilSettled()

  5349. 

  5350.         self.assertEqual(0, len(A.messages))

  5351. 

  5352.     @simple_layout('layouts/merge-failure.yaml')

  5353.     def test_merge_failure_reporters(self):

  5354.         """Check that the config is set up correctly"""

  5355. 

  5356.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5357.         self.assertEqual(

  5358.             "Merge Failed.\n\nThis change or one of its cross-repo "

  5359.             "dependencies was unable to be automatically merged with the "

  5360.             "current state of its repository. Please rebase the change and "

  5361.             "upload a new patchset.",

  5362.             tenant.layout.pipelines['check'].merge_failure_message)

  5363.         self.assertEqual(

  5364.             "The merge failed! For more information...",

  5365.             tenant.layout.pipelines['gate'].merge_failure_message)

  5366. 

  5367.         self.assertEqual(

  5368.             len(tenant.layout.pipelines['check'].merge_failure_actions), 1)

  5369.         self.assertEqual(

  5370.             len(tenant.layout.pipelines['gate'].merge_failure_actions), 2)

  5371. 

  5372.         self.assertTrue(isinstance(

  5373.             tenant.layout.pipelines['check'].merge_failure_actions[0],

  5374.             gerritreporter.GerritReporter))

  5375. 

  5376.         self.assertTrue(

  5377.             (

  5378.                 isinstance(tenant.layout.pipelines['gate'].

  5379.                            merge_failure_actions[0],

  5380.                            zuul.driver.smtp.smtpreporter.SMTPReporter) and

  5381.                 isinstance(tenant.layout.pipelines['gate'].

  5382.                            merge_failure_actions[1],

  5383.                            gerritreporter.GerritReporter)

  5384.             ) or (

  5385.                 isinstance(tenant.layout.pipelines['gate'].

  5386.                            merge_failure_actions[0],

  5387.                            gerritreporter.GerritReporter) and

  5388.                 isinstance(tenant.layout.pipelines['gate'].

  5389.                            merge_failure_actions[1],

  5390.                            zuul.driver.smtp.smtpreporter.SMTPReporter)

  5391.             )

  5392.         )

  5393. 

  5394.     @skip("Disabled for early v3 development")

  5395.     def test_merge_failure_reports(self):

  5396.         """Check that when a change fails to merge the correct message is sent

  5397.         to the correct reporter"""

  5398.         self.updateConfigLayout(

  5399.             'tests/fixtures/layout-merge-failure.yaml')

  5400.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  5401.         self.registerJobs()

  5402. 

  5403.         # Check a test failure isn't reported to SMTP

  5404.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5405.         A.addApproval('Code-Review', 2)

  5406.         self.executor_server.failJob('project-test1', A)

  5407.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5408.         self.waitUntilSettled()

  5409. 

  5410.         self.assertEqual(3, len(self.history))  # 3 jobs

  5411.         self.assertEqual(0, len(self.smtp_messages))

  5412. 

  5413.         # Check a merge failure is reported to SMTP

  5414.         # B should be merged, but C will conflict with B

  5415.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  5416.         B.addPatchset(['conflict'])

  5417.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  5418.         C.addPatchset(['conflict'])

  5419.         B.addApproval('Code-Review', 2)

  5420.         C.addApproval('Code-Review', 2)

  5421.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  5422.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  5423.         self.waitUntilSettled()

  5424. 

  5425.         self.assertEqual(6, len(self.history))  # A and B jobs

  5426.         self.assertEqual(1, len(self.smtp_messages))

  5427.         self.assertEqual('The merge failed! For more information...',

  5428.                          self.smtp_messages[0]['body'])

  5429. 

  5430.     @skip("Disabled for early v3 development")

  5431.     def test_default_merge_failure_reports(self):

  5432.         """Check that the default merge failure reports are correct."""

  5433. 

  5434.         # A should report success, B should report merge failure.

  5435.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5436.         A.addPatchset(['conflict'])

  5437.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  5438.         B.addPatchset(['conflict'])

  5439.         A.addApproval('Code-Review', 2)

  5440.         B.addApproval('Code-Review', 2)

  5441.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5442.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  5443.         self.waitUntilSettled()

  5444. 

  5445.         self.assertEqual(3, len(self.history))  # A jobs

  5446.         self.assertEqual(A.reported, 2)

  5447.         self.assertEqual(B.reported, 2)

  5448.         self.assertEqual(A.data['status'], 'MERGED')

  5449.         self.assertEqual(B.data['status'], 'NEW')

  5450.         self.assertIn('Build succeeded', A.messages[1])

  5451.         self.assertIn('Merge Failed', B.messages[1])

  5452.         self.assertIn('automatically merged', B.messages[1])

  5453.         self.assertNotIn('logs.example.com', B.messages[1])

  5454.         self.assertNotIn('SKIPPED', B.messages[1])

  5455. 

  5456.     def test_client_get_running_jobs(self):

  5457.         "Test that the RPC client can get a list of running jobs"

  5458.         self.executor_server.hold_jobs_in_build = True

  5459.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5460.         A.addApproval('Code-Review', 2)

  5461.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5462.         self.waitUntilSettled()

  5463. 

  5464.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  5465.                                           self.gearman_server.port)

  5466.         self.addCleanup(client.shutdown)

  5467. 

  5468.         # Wait for gearman server to send the initial workData back to zuul

  5469.         start = time.time()

  5470.         while True:

  5471.             if time.time() - start > 10:

  5472.                 raise Exception("Timeout waiting for gearman server to report "

  5473.                                 + "back to the client")

  5474.             build = list(self.scheds.first.sched.executor.builds.values())[0]

  5475.             if build.worker.name == self.executor_server.hostname:

  5476.                 break

  5477.             else:

  5478.                 time.sleep(0)

  5479. 

  5480.         running_items = client.get_running_jobs()

  5481. 

  5482.         self.assertEqual(1, len(running_items))

  5483.         running_item = running_items[0]

  5484.         self.assertEqual([], running_item['failing_reasons'])

  5485.         self.assertEqual([], running_item['items_behind'])

  5486.         self.assertEqual('https://review.example.com/1', running_item['url'])

  5487.         self.assertIsNone(running_item['item_ahead'])

  5488.         self.assertEqual('org/project', running_item['project'])

  5489.         self.assertIsNone(running_item['remaining_time'])

  5490.         self.assertEqual(True, running_item['active'])

  5491.         self.assertEqual('1,1', running_item['id'])

  5492. 

  5493.         self.assertEqual(3, len(running_item['jobs']))

  5494.         for job in running_item['jobs']:

  5495.             if job['name'] == 'project-merge':

  5496.                 self.assertEqual('project-merge', job['name'])

  5497.                 self.assertEqual('gate', job['pipeline'])

  5498.                 self.assertEqual(False, job['retry'])

  5499.                 self.assertEqual(

  5500.                     'stream/{uuid}?logfile=console.log'

  5501.                     .format(uuid=job['uuid']), job['url'])

  5502.                 self.assertEqual(

  5503.                     'finger://{hostname}/{uuid}'.format(

  5504.                         hostname=self.executor_server.hostname,

  5505.                         uuid=job['uuid']),

  5506.                     job['finger_url'])

  5507.                 self.assertEqual(2, len(job['worker']))

  5508.                 self.assertEqual(False, job['canceled'])

  5509.                 self.assertEqual(True, job['voting'])

  5510.                 self.assertIsNone(job['result'])

  5511.                 self.assertEqual('gate', job['pipeline'])

  5512.                 break

  5513. 

  5514.         self.executor_server.hold_jobs_in_build = False

  5515.         self.executor_server.release()

  5516.         self.waitUntilSettled()

  5517. 

  5518.         running_items = client.get_running_jobs()

  5519.         self.assertEqual(0, len(running_items))

  5520. 

  5521.     @simple_layout('layouts/nonvoting-pipeline.yaml')

  5522.     def test_nonvoting_pipeline(self):

  5523.         "Test that a nonvoting pipeline (experimental) can still report"

  5524. 

  5525.         A = self.fake_gerrit.addFakeChange('org/experimental-project',

  5526.                                            'master', 'A')

  5527.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  5528.         self.waitUntilSettled()

  5529. 

  5530.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  5531.                          'SUCCESS')

  5532.         self.assertEqual(

  5533.             self.getJobFromHistory('experimental-project-test').result,

  5534.             'SUCCESS')

  5535.         self.assertEqual(A.reported, 1)

  5536. 

  5537.     @simple_layout('layouts/disable_at.yaml')

  5538.     def test_disable_at(self):

  5539.         "Test a pipeline will only report to the disabled trigger when failing"

  5540. 

  5541.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5542.         self.assertEqual(3, tenant.layout.pipelines['check'].disable_at)

  5543.         self.assertEqual(

  5544.             0, tenant.layout.pipelines['check']._consecutive_failures)

  5545.         self.assertFalse(tenant.layout.pipelines['check']._disabled)

  5546. 

  5547.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5548.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  5549.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  5550.         D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')

  5551.         E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')

  5552.         F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')

  5553.         G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')

  5554.         H = self.fake_gerrit.addFakeChange('org/project', 'master', 'H')

  5555.         I = self.fake_gerrit.addFakeChange('org/project', 'master', 'I')

  5556.         J = self.fake_gerrit.addFakeChange('org/project', 'master', 'J')

  5557.         K = self.fake_gerrit.addFakeChange('org/project', 'master', 'K')

  5558. 

  5559.         self.executor_server.failJob('project-test1', A)

  5560.         self.executor_server.failJob('project-test1', B)

  5561.         # Let C pass, resetting the counter

  5562.         self.executor_server.failJob('project-test1', D)

  5563.         self.executor_server.failJob('project-test1', E)

  5564.         self.executor_server.failJob('project-test1', F)

  5565.         self.executor_server.failJob('project-test1', G)

  5566.         self.executor_server.failJob('project-test1', H)

  5567.         # I also passes but should only report to the disabled reporters

  5568.         self.executor_server.failJob('project-test1', J)

  5569.         self.executor_server.failJob('project-test1', K)

  5570. 

  5571.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  5572.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  5573.         self.waitUntilSettled()

  5574. 

  5575.         self.assertEqual(

  5576.             2, tenant.layout.pipelines['check']._consecutive_failures)

  5577.         self.assertFalse(tenant.layout.pipelines['check']._disabled)

  5578. 

  5579.         self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

  5580.         self.waitUntilSettled()

  5581. 

  5582.         self.assertEqual(

  5583.             0, tenant.layout.pipelines['check']._consecutive_failures)

  5584.         self.assertFalse(tenant.layout.pipelines['check']._disabled)

  5585. 

  5586.         self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))

  5587.         self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))

  5588.         self.fake_gerrit.addEvent(F.getPatchsetCreatedEvent(1))

  5589.         self.waitUntilSettled()

  5590. 

  5591.         # We should be disabled now

  5592.         self.assertEqual(

  5593.             3, tenant.layout.pipelines['check']._consecutive_failures)

  5594.         self.assertTrue(tenant.layout.pipelines['check']._disabled)

  5595. 

  5596.         # We need to wait between each of these patches to make sure the

  5597.         # smtp messages come back in an expected order

  5598.         self.fake_gerrit.addEvent(G.getPatchsetCreatedEvent(1))

  5599.         self.waitUntilSettled()

  5600.         self.fake_gerrit.addEvent(H.getPatchsetCreatedEvent(1))

  5601.         self.waitUntilSettled()

  5602.         self.fake_gerrit.addEvent(I.getPatchsetCreatedEvent(1))

  5603.         self.waitUntilSettled()

  5604. 

  5605.         # The first 6 (ABCDEF) jobs should have reported back to gerrt thus

  5606.         # leaving a message on each change

  5607.         self.assertEqual(1, len(A.messages))

  5608.         self.assertIn('Build failed.', A.messages[0])

  5609.         self.assertEqual(1, len(B.messages))

  5610.         self.assertIn('Build failed.', B.messages[0])

  5611.         self.assertEqual(1, len(C.messages))

  5612.         self.assertIn('Build succeeded.', C.messages[0])

  5613.         self.assertEqual(1, len(D.messages))

  5614.         self.assertIn('Build failed.', D.messages[0])

  5615.         self.assertEqual(1, len(E.messages))

  5616.         self.assertIn('Build failed.', E.messages[0])

  5617.         self.assertEqual(1, len(F.messages))

  5618.         self.assertIn('Build failed.', F.messages[0])

  5619. 

  5620.         # The last 3 (GHI) would have only reported via smtp.

  5621.         self.assertEqual(3, len(self.smtp_messages))

  5622.         self.assertEqual(0, len(G.messages))

  5623.         self.assertIn('Build failed.', self.smtp_messages[0]['body'])

  5624.         self.assertIn(

  5625.             'project-test1 finger://', self.smtp_messages[0]['body'])

  5626.         self.assertEqual(0, len(H.messages))

  5627.         self.assertIn('Build failed.', self.smtp_messages[1]['body'])

  5628.         self.assertIn(

  5629.             'project-test1 finger://', self.smtp_messages[1]['body'])

  5630.         self.assertEqual(0, len(I.messages))

  5631.         self.assertIn('Build succeeded.', self.smtp_messages[2]['body'])

  5632.         self.assertIn(

  5633.             'project-test1 finger://', self.smtp_messages[2]['body'])

  5634. 

  5635.         # Now reload the configuration (simulate a HUP) to check the pipeline

  5636.         # comes out of disabled

  5637.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  5638. 

  5639.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5640. 

  5641.         self.assertEqual(3, tenant.layout.pipelines['check'].disable_at)

  5642.         self.assertEqual(

  5643.             0, tenant.layout.pipelines['check']._consecutive_failures)

  5644.         self.assertFalse(tenant.layout.pipelines['check']._disabled)

  5645. 

  5646.         self.fake_gerrit.addEvent(J.getPatchsetCreatedEvent(1))

  5647.         self.fake_gerrit.addEvent(K.getPatchsetCreatedEvent(1))

  5648.         self.waitUntilSettled()

  5649. 

  5650.         self.assertEqual(

  5651.             2, tenant.layout.pipelines['check']._consecutive_failures)

  5652.         self.assertFalse(tenant.layout.pipelines['check']._disabled)

  5653. 

  5654.         # J and K went back to gerrit

  5655.         self.assertEqual(1, len(J.messages))

  5656.         self.assertIn('Build failed.', J.messages[0])

  5657.         self.assertEqual(1, len(K.messages))

  5658.         self.assertIn('Build failed.', K.messages[0])

  5659.         # No more messages reported via smtp

  5660.         self.assertEqual(3, len(self.smtp_messages))

  5661. 

  5662.     @simple_layout('layouts/one-job-project.yaml')

  5663.     def test_one_job_project(self):

  5664.         "Test that queueing works with one job"

  5665.         A = self.fake_gerrit.addFakeChange('org/one-job-project',

  5666.                                            'master', 'A')

  5667.         B = self.fake_gerrit.addFakeChange('org/one-job-project',

  5668.                                            'master', 'B')

  5669.         A.addApproval('Code-Review', 2)

  5670.         B.addApproval('Code-Review', 2)

  5671.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5672.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  5673.         self.waitUntilSettled()

  5674. 

  5675.         self.assertEqual(A.data['status'], 'MERGED')

  5676.         self.assertEqual(A.reported, 2)

  5677.         self.assertEqual(B.data['status'], 'MERGED')

  5678.         self.assertEqual(B.reported, 2)

  5679. 

  5680.     def test_job_aborted(self):

  5681.         "Test that if a execute server aborts a job, it is run again"

  5682.         self.executor_server.hold_jobs_in_build = True

  5683.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5684.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  5685.         self.waitUntilSettled()

  5686. 

  5687.         self.executor_server.release('.*-merge')

  5688.         self.waitUntilSettled()

  5689. 

  5690.         self.assertEqual(len(self.builds), 2)

  5691. 

  5692.         # first abort

  5693.         self.builds[0].aborted = True

  5694.         self.executor_server.release('.*-test*')

  5695.         self.waitUntilSettled()

  5696.         self.assertEqual(len(self.builds), 1)

  5697. 

  5698.         # second abort

  5699.         self.builds[0].aborted = True

  5700.         self.executor_server.release('.*-test*')

  5701.         self.waitUntilSettled()

  5702.         self.assertEqual(len(self.builds), 1)

  5703. 

  5704.         # third abort

  5705.         self.builds[0].aborted = True

  5706.         self.executor_server.release('.*-test*')

  5707.         self.waitUntilSettled()

  5708.         self.assertEqual(len(self.builds), 1)

  5709. 

  5710.         # fourth abort

  5711.         self.builds[0].aborted = True

  5712.         self.executor_server.release('.*-test*')

  5713.         self.waitUntilSettled()

  5714.         self.assertEqual(len(self.builds), 1)

  5715. 

  5716.         self.executor_server.hold_jobs_in_build = False

  5717.         self.executor_server.release()

  5718.         self.waitUntilSettled()

  5719. 

  5720.         self.assertEqual(len(self.history), 7)

  5721.         self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)

  5722.         self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)

  5723. 

  5724.     def test_rerun_on_abort(self):

  5725.         "Test that if a execute server fails to run a job, it is run again"

  5726. 

  5727.         self.executor_server.hold_jobs_in_build = True

  5728.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5729.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  5730.         self.waitUntilSettled()

  5731. 

  5732.         self.executor_server.release('.*-merge')

  5733.         self.waitUntilSettled()

  5734. 

  5735.         self.assertEqual(len(self.builds), 2)

  5736.         self.builds[0].requeue = True

  5737.         self.executor_server.release('.*-test*')

  5738.         self.waitUntilSettled()

  5739. 

  5740.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  5741.         items = tenant.layout.pipelines['check'].getAllItems()

  5742.         build_set = items[0].current_build_set

  5743. 

  5744.         for x in range(3):

  5745.             # We should have x+1 retried builds for project-test1

  5746.             retry_builds = build_set.getRetryBuildsForJob('project-test1')

  5747.             self.assertEqual(len(retry_builds), x + 1)

  5748.             for build in retry_builds:

  5749.                 self.assertEqual(build.retry, True)

  5750.                 self.assertEqual(build.result, 'RETRY')

  5751. 

  5752.             self.assertEqual(len(self.builds), 1,

  5753.                              'len of builds at x=%d is wrong' % x)

  5754.             self.builds[0].requeue = True

  5755.             self.executor_server.release('.*-test1')

  5756.             self.waitUntilSettled()

  5757. 

  5758.         self.executor_server.hold_jobs_in_build = False

  5759.         self.executor_server.release()

  5760.         self.waitUntilSettled()

  5761.         self.assertEqual(len(self.history), 6)

  5762.         self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 2)

  5763.         self.assertEqual(A.reported, 1)

  5764.         self.assertIn('RETRY_LIMIT', A.messages[0])

  5765. 

  5766.     def test_zookeeper_disconnect(self):

  5767.         "Test that jobs are executed after a zookeeper disconnect"

  5768. 

  5769.         self.fake_nodepool.pause()

  5770.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5771.         A.addApproval('Code-Review', 2)

  5772.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5773.         self.waitUntilSettled()

  5774. 

  5775.         self.scheds.execute(lambda app: app.sched.zk.client.stop())

  5776.         self.scheds.execute(lambda app: app.sched.zk.client.start())

  5777.         self.fake_nodepool.unpause()

  5778.         self.waitUntilSettled()

  5779. 

  5780.         self.assertEqual(A.data['status'], 'MERGED')

  5781.         self.assertEqual(A.reported, 2)

  5782. 

  5783.     def test_zookeeper_disconnect2(self):

  5784.         "Test that jobs are executed after a zookeeper disconnect"

  5785. 

  5786.         # This tests receiving a ZK disconnect between the arrival of

  5787.         # a fulfilled request and when we accept its nodes.

  5788.         self.fake_nodepool.pause()

  5789.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5790.         A.addApproval('Code-Review', 2)

  5791.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5792.         self.waitUntilSettled()

  5793. 

  5794.         # We're waiting on the nodepool request to complete.  Stop the

  5795.         # scheduler from processing further events, then fulfill the

  5796.         # nodepool request.

  5797.         self.scheds.first.sched.run_handler_lock.acquire()

  5798. 

  5799.         # Fulfill the nodepool request.

  5800.         self.fake_nodepool.unpause()

  5801.         requests = list(self.scheds.first.sched.nodepool.requests.values())

  5802.         self.assertEqual(1, len(requests))

  5803.         request = requests[0]

  5804.         for x in iterate_timeout(30, 'fulfill request'):

  5805.             if request.fulfilled:

  5806.                 break

  5807.         id1 = request.id

  5808. 

  5809.         # The request is fulfilled, but the scheduler hasn't processed

  5810.         # it yet.  Reconnect ZK.

  5811.         self.scheds.execute(lambda app: app.sched.zk.client.stop())

  5812.         self.scheds.execute(lambda app: app.sched.zk.client.start())

  5813. 

  5814.         # Allow the scheduler to continue and process the (now

  5815.         # out-of-date) notification that nodes are ready.

  5816.         self.scheds.first.sched.run_handler_lock.release()

  5817. 

  5818.         # It should resubmit the request, once it's fulfilled, we can

  5819.         # wait for it to run jobs and settle.

  5820.         for x in iterate_timeout(30, 'fulfill request'):

  5821.             if request.fulfilled:

  5822.                 break

  5823.         self.waitUntilSettled()

  5824. 

  5825.         id2 = request.id

  5826.         self.assertEqual(A.data['status'], 'MERGED')

  5827.         self.assertEqual(A.reported, 2)

  5828.         # Make sure it was resubmitted (the id's should be different).

  5829.         self.assertNotEqual(id1, id2)

  5830. 

  5831.     def test_nodepool_failure(self):

  5832.         "Test that jobs are reported after a nodepool failure"

  5833. 

  5834.         self.fake_nodepool.pause()

  5835.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5836.         A.addApproval('Code-Review', 2)

  5837.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5838.         self.waitUntilSettled()

  5839. 

  5840.         req = self.fake_nodepool.getNodeRequests()[0]

  5841.         self.fake_nodepool.addFailRequest(req)

  5842. 

  5843.         self.fake_nodepool.unpause()

  5844.         self.waitUntilSettled()

  5845. 

  5846.         self.assertEqual(A.data['status'], 'NEW')

  5847.         self.assertEqual(A.reported, 2)

  5848.         self.assertIn('project-merge : NODE_FAILURE', A.messages[1])

  5849.         self.assertIn('project-test1 : SKIPPED', A.messages[1])

  5850.         self.assertIn('project-test2 : SKIPPED', A.messages[1])

  5851. 

  5852.     def test_nodepool_resources(self):

  5853.         "Test that resources are reported"

  5854. 

  5855.         self.executor_server.hold_jobs_in_build = True

  5856.         self.fake_nodepool.resources = {

  5857.             'cores': 2,

  5858.             'ram': 1024,

  5859.             'instances': 1,

  5860.         }

  5861.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5862.         A.addApproval('Code-Review', 2)

  5863.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  5864.         self.waitUntilSettled()

  5865. 

  5866.         self.executor_server.release('project-merge')

  5867.         self.waitUntilSettled()

  5868. 

  5869.         # Check that resource usage gauges are reported

  5870.         self.assertHistory([

  5871.             dict(name='project-merge', result='SUCCESS', changes='1,1'),

  5872.         ])

  5873.         self.assertReportedStat(

  5874.             'zuul.nodepool.resources.tenant.tenant-one.cores',

  5875.             value='2', kind='g')

  5876.         self.assertReportedStat(

  5877.             'zuul.nodepool.resources.tenant.tenant-one.ram',

  5878.             value='1024', kind='g')

  5879.         self.assertReportedStat(

  5880.             'zuul.nodepool.resources.tenant.tenant-one.instances',

  5881.             value='1', kind='g')

  5882.         self.assertReportedStat(

  5883.             'zuul.nodepool.resources.project.review_example_com/org/project.'

  5884.             'cores', value='2', kind='g')

  5885.         self.assertReportedStat(

  5886.             'zuul.nodepool.resources.project.review_example_com/org/project.'

  5887.             'ram', value='1024', kind='g')

  5888.         self.assertReportedStat(

  5889.             'zuul.nodepool.resources.project.review_example_com/org/project.'

  5890.             'instances', value='1', kind='g')

  5891. 

  5892.         # Check that resource usage counters are reported

  5893.         self.assertReportedStat(

  5894.             'zuul.nodepool.resources.tenant.tenant-one.cores',

  5895.             kind='c')

  5896.         self.assertReportedStat(

  5897.             'zuul.nodepool.resources.tenant.tenant-one.ram',

  5898.             kind='c')

  5899.         self.assertReportedStat(

  5900.             'zuul.nodepool.resources.tenant.tenant-one.instances',

  5901.             kind='c')

  5902.         self.assertReportedStat(

  5903.             'zuul.nodepool.resources.project.review_example_com/org/project.'

  5904.             'cores', kind='c')

  5905.         self.assertReportedStat(

  5906.             'zuul.nodepool.resources.project.review_example_com/org/project.'

  5907.             'ram', kind='c')

  5908.         self.assertReportedStat(

  5909.             'zuul.nodepool.resources.project.review_example_com/org/project.'

  5910.             'instances', kind='c')

  5911. 

  5912.         self.executor_server.hold_jobs_in_build = False

  5913.         self.executor_server.release()

  5914.         self.waitUntilSettled()

  5915.         self.assertEqual(A.data['status'], 'MERGED')

  5916.         self.assertEqual(A.reported, 2)

  5917. 

  5918.     def test_nodepool_pipeline_priority(self):

  5919.         "Test that nodes are requested at the correct pipeline priority"

  5920. 

  5921.         self.fake_nodepool.pause()

  5922. 

  5923.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5924.         self.fake_gerrit.addEvent(A.getRefUpdatedEvent())

  5925.         self.waitUntilSettled()

  5926. 

  5927.         B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')

  5928.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  5929.         self.waitUntilSettled()

  5930. 

  5931.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  5932.         C.addApproval('Code-Review', 2)

  5933.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  5934.         self.waitUntilSettled()

  5935. 

  5936.         reqs = self.fake_nodepool.getNodeRequests()

  5937. 

  5938.         # The requests come back sorted by priority. Since we have

  5939.         # three requests for the three changes each with a different

  5940.         # priority.  Also they get a serial number based on order they

  5941.         # were received so the number on the endof the oid should map

  5942.         # to order submitted.

  5943. 

  5944.         # * gate first - high priority - change C

  5945.         self.assertEqual(reqs[0]['_oid'], '100-0000000002')

  5946.         self.assertEqual(reqs[0]['node_types'], ['label1'])

  5947.         # * check second - normal priority - change B

  5948.         self.assertEqual(reqs[1]['_oid'], '200-0000000001')

  5949.         self.assertEqual(reqs[1]['node_types'], ['label1'])

  5950.         # * post third - low priority - change A

  5951.         # additionally, the post job defined uses an ubuntu-xenial node,

  5952.         # so we include that check just as an extra verification

  5953.         self.assertEqual(reqs[2]['_oid'], '300-0000000000')

  5954.         self.assertEqual(reqs[2]['node_types'], ['ubuntu-xenial'])

  5955. 

  5956.         self.fake_nodepool.unpause()

  5957.         self.waitUntilSettled()

  5958. 

  5959.     @simple_layout('layouts/two-projects-integrated.yaml')

  5960.     def test_nodepool_relative_priority_check(self):

  5961.         "Test that nodes are requested at the relative priority"

  5962. 

  5963.         self.fake_nodepool.pause()

  5964. 

  5965.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  5966.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  5967.         self.waitUntilSettled()

  5968. 

  5969.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  5970.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  5971.         self.waitUntilSettled()

  5972. 

  5973.         C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')

  5974.         self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

  5975.         self.waitUntilSettled()

  5976. 

  5977.         D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')

  5978.         self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))

  5979.         self.waitUntilSettled()

  5980. 

  5981.         reqs = self.fake_nodepool.getNodeRequests()

  5982. 

  5983.         # The requests come back sorted by priority.

  5984. 

  5985.         # Change A, first change for project, high relative priority.

  5986.         self.assertEqual(reqs[0]['_oid'], '200-0000000000')

  5987.         self.assertEqual(reqs[0]['relative_priority'], 0)

  5988. 

  5989.         # Change C, first change for project1, high relative priority.

  5990.         self.assertEqual(reqs[1]['_oid'], '200-0000000002')

  5991.         self.assertEqual(reqs[1]['relative_priority'], 0)

  5992. 

  5993.         # Change B, second change for project, lower relative priority.

  5994.         self.assertEqual(reqs[2]['_oid'], '200-0000000001')

  5995.         self.assertEqual(reqs[2]['relative_priority'], 1)

  5996. 

  5997.         # Change D, first change for project2 shared with project1,

  5998.         # lower relative priority than project1.

  5999.         self.assertEqual(reqs[3]['_oid'], '200-0000000003')

  6000.         self.assertEqual(reqs[3]['relative_priority'], 1)

  6001. 

  6002.         # Fulfill only the first request

  6003.         self.fake_nodepool.fulfillRequest(reqs[0])

  6004.         for x in iterate_timeout(30, 'fulfill request'):

  6005.             if len(self.scheds.first.sched.nodepool.requests) < 4:

  6006.                 break

  6007.         self.waitUntilSettled()

  6008. 

  6009.         reqs = self.fake_nodepool.getNodeRequests()

  6010. 

  6011.         # Change B, now first change for project, equal priority.

  6012.         self.assertEqual(reqs[0]['_oid'], '200-0000000001')

  6013.         self.assertEqual(reqs[0]['relative_priority'], 0)

  6014. 

  6015.         # Change C, now first change for project1, equal priority.

  6016.         self.assertEqual(reqs[1]['_oid'], '200-0000000002')

  6017.         self.assertEqual(reqs[1]['relative_priority'], 0)

  6018. 

  6019.         # Change D, first change for project2 shared with project1,

  6020.         # still lower relative priority than project1.

  6021.         self.assertEqual(reqs[2]['_oid'], '200-0000000003')

  6022.         self.assertEqual(reqs[2]['relative_priority'], 1)

  6023. 

  6024.         self.fake_nodepool.unpause()

  6025.         self.waitUntilSettled()

  6026. 

  6027.     @simple_layout('layouts/two-projects-integrated.yaml')

  6028.     def test_nodepool_relative_priority_gate(self):

  6029.         "Test that nodes are requested at the relative priority"

  6030. 

  6031.         self.fake_nodepool.pause()

  6032. 

  6033.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  6034.         A.addApproval('Code-Review', 2)

  6035.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  6036.         self.waitUntilSettled()

  6037. 

  6038.         B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')

  6039.         B.addApproval('Code-Review', 2)

  6040.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  6041.         self.waitUntilSettled()

  6042. 

  6043.         # project does not share a queue with project1 and project2.

  6044.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')

  6045.         C.addApproval('Code-Review', 2)

  6046.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  6047.         self.waitUntilSettled()

  6048. 

  6049.         reqs = self.fake_nodepool.getNodeRequests()

  6050. 

  6051.         # The requests come back sorted by priority.

  6052. 

  6053.         # Change A, first change for shared queue, high relative

  6054.         # priority.

  6055.         self.assertEqual(reqs[0]['_oid'], '100-0000000000')

  6056.         self.assertEqual(reqs[0]['relative_priority'], 0)

  6057. 

  6058.         # Change C, first change for independent project, high

  6059.         # relative priority.

  6060.         self.assertEqual(reqs[1]['_oid'], '100-0000000002')

  6061.         self.assertEqual(reqs[1]['relative_priority'], 0)

  6062. 

  6063.         # Change B, second change for shared queue, lower relative

  6064.         # priority.

  6065.         self.assertEqual(reqs[2]['_oid'], '100-0000000001')

  6066.         self.assertEqual(reqs[2]['relative_priority'], 1)

  6067. 

  6068.         self.fake_nodepool.unpause()

  6069.         self.waitUntilSettled()

  6070. 

  6071.     def test_nodepool_job_removal(self):

  6072.         "Test that nodes are returned unused after job removal"

  6073. 

  6074.         self.fake_nodepool.pause()

  6075.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  6076.         A.addApproval('Code-Review', 2)

  6077.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  6078.         self.waitUntilSettled()

  6079. 

  6080.         self.commitConfigUpdate('common-config', 'layouts/no-jobs.yaml')

  6081.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  6082.         self.waitUntilSettled()

  6083. 

  6084.         self.fake_nodepool.unpause()

  6085.         self.waitUntilSettled()

  6086. 

  6087.         self.assertEqual(A.data['status'], 'MERGED')

  6088.         self.assertEqual(A.reported, 2)

  6089.         self.assertHistory([

  6090.             dict(name='gate-noop', result='SUCCESS', changes='1,1'),

  6091.         ])

  6092.         for node in self.fake_nodepool.getNodes():

  6093.             self.assertFalse(node['_lock'])

  6094.             self.assertEqual(node['state'], 'ready')

  6095. 

  6096.     @simple_layout('layouts/multiple-templates.yaml')

  6097.     def test_multiple_project_templates(self):

  6098.         # Test that applying multiple project templates to a project

  6099.         # doesn't alter them when used for a second project.

  6100.         A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')

  6101.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6102.         self.waitUntilSettled()

  6103. 

  6104.         build = self.getJobFromHistory('py27')

  6105.         self.assertEqual(build.parameters['zuul']['jobtags'], [])

  6106. 

  6107.     def test_pending_merge_in_reconfig(self):

  6108.         # Test that if we are waiting for an outstanding merge on

  6109.         # reconfiguration that we continue to do so.

  6110.         self.gearman_server.hold_merge_jobs_in_queue = True

  6111.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  6112.         A.setMerged()

  6113.         self.fake_gerrit.addEvent(A.getRefUpdatedEvent())

  6114.         self.waitUntilSettled()

  6115. 

  6116.         self.assertEqual(len(self.scheds.first.sched.merger.jobs), 1)

  6117.         gearJob = next(iter(self.scheds.first.sched.merger.jobs))

  6118.         self.assertEqual(gearJob.complete, False)

  6119. 

  6120.         # Reconfigure while we still have an outstanding merge job

  6121.         self.gearman_server.hold_merge_jobs_in_queue = False

  6122.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  6123.         (trusted, project1) = tenant.getProject('org/project1')

  6124.         self.scheds.first.sched.reconfigureTenant(

  6125.             self.scheds.first.sched.abide.tenants['tenant-one'],

  6126.             project1, None)

  6127.         self.waitUntilSettled()

  6128. 

  6129.         # Verify the merge job is still running and that the item is

  6130.         # in the pipeline

  6131.         self.assertEqual(gearJob.complete, False)

  6132.         self.assertEqual(len(self.scheds.first.sched.merger.jobs), 1)

  6133. 

  6134.         pipeline = tenant.layout.pipelines['post']

  6135.         self.assertEqual(len(pipeline.getAllItems()), 1)

  6136.         self.gearman_server.release()

  6137.         self.waitUntilSettled()

  6138. 

  6139.         self.assertEqual(gearJob.complete, True)

  6140.         self.assertEqual(len(self.scheds.first.sched.merger.jobs), 0)

  6141. 

  6142.     @simple_layout('layouts/parent-matchers.yaml')

  6143.     def test_parent_matchers(self):

  6144.         "Test that if a job's parent does not match, the job does not run"

  6145.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  6146.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6147.         self.waitUntilSettled()

  6148. 

  6149.         self.assertHistory([])

  6150. 

  6151.         files = {'foo.txt': ''}

  6152.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',

  6153.                                            files=files)

  6154.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  6155.         self.waitUntilSettled()

  6156. 

  6157.         files = {'bar.txt': ''}

  6158.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C',

  6159.                                            files=files)

  6160.         self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

  6161.         self.waitUntilSettled()

  6162. 

  6163.         files = {'foo.txt': '', 'bar.txt': ''}

  6164.         D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D',

  6165.                                            files=files)

  6166.         self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))

  6167.         self.waitUntilSettled()

  6168. 

  6169.         self.assertHistory([

  6170.             dict(name='child-job', result='SUCCESS', changes='3,1'),

  6171.             dict(name='child-job', result='SUCCESS', changes='4,1'),

  6172.         ], ordered=False)

  6173. 

  6174.     @simple_layout('layouts/file-matchers.yaml')

  6175.     def test_file_matchers(self):

  6176.         "Test several file matchers"

  6177.         files = {'parent1.txt': ''}

  6178.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',

  6179.                                            files=files)

  6180.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6181.         self.waitUntilSettled()

  6182. 

  6183.         files = {'parent2.txt': ''}

  6184.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',

  6185.                                            files=files)

  6186.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  6187.         self.waitUntilSettled()

  6188. 

  6189.         files = {'child.txt': ''}

  6190.         C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C',

  6191.                                            files=files)

  6192.         self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

  6193.         self.waitUntilSettled()

  6194. 

  6195.         files = {'project.txt': ''}

  6196.         D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D',

  6197.                                            files=files)

  6198.         self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))

  6199.         self.waitUntilSettled()

  6200. 

  6201.         files = {'tests/foo': ''}

  6202.         E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E',

  6203.                                            files=files)

  6204.         self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))

  6205.         self.waitUntilSettled()

  6206. 

  6207.         files = {'tests/docs/foo': ''}

  6208.         F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F',

  6209.                                            files=files)

  6210.         self.fake_gerrit.addEvent(F.getPatchsetCreatedEvent(1))

  6211.         self.waitUntilSettled()

  6212. 

  6213.         self.assertHistory([

  6214.             dict(name='child-job', result='SUCCESS', changes='2,1'),

  6215. 

  6216.             dict(name='child-override-job', result='SUCCESS', changes='3,1'),

  6217. 

  6218.             dict(name='project-override-job', result='SUCCESS', changes='4,1'),

  6219. 

  6220.             dict(name='irr-job', result='SUCCESS', changes='5,1'),

  6221.             dict(name='irr-override-job', result='SUCCESS', changes='5,1'),

  6222. 

  6223.             dict(name='irr-job', result='SUCCESS', changes='6,1'),

  6224.         ], ordered=False)

  6225. 

  6226.     def test_trusted_project_dep_on_non_live_untrusted_project(self):

  6227.         # Test we get a layout for trusted projects when they depend on

  6228.         # non live untrusted projects. This checks against a bug where

  6229.         # trusted project config changes can end up in a infinite loop

  6230.         # trying to find the right layout.

  6231.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  6232.         files = {'zuul.yaml': ''}

  6233.         B = self.fake_gerrit.addFakeChange('common-config', 'master', 'B',

  6234.                                            files=files)

  6235.         B.setDependsOn(A, 1)

  6236.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  6237.         self.waitUntilSettled()

  6238. 

  6239.         self.assertHistory([

  6240.             dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),

  6241.             dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'),

  6242.             dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'),

  6243.             dict(name='project1-project2-integration',

  6244.                  result='SUCCESS', changes='1,1 2,1'),

  6245.         ], ordered=False)

  6246. 

  6247. 

  6248. class TestJobUpdateBrokenConfig(ZuulTestCase):

  6249.     tenant_config_file = 'config/job-update-broken/main.yaml'

  6250. 

  6251.     def test_fix_check_without_running(self):

  6252.         "Test that we can fix a broken check pipeline (don't run the job)"

  6253.         in_repo_conf = textwrap.dedent(

  6254.             """

  6255.             - job:

  6256.                 name: existing-files

  6257.                 files:

  6258.                   - README.txt

  6259. 

  6260.             - project-template:

  6261.                 name: files-template

  6262.                 check:

  6263.                   jobs:

  6264.                     - existing-files

  6265.                     - noop

  6266.             """)

  6267. 

  6268.         # When the config is broken, we don't override any files

  6269.         # matchers since we don't have a valid basis.  Since this

  6270.         # doesn't update README.txt, nothing should run.

  6271.         file_dict = {'zuul.d/existing.yaml': in_repo_conf}

  6272.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',

  6273.                                            files=file_dict)

  6274.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6275.         self.waitUntilSettled()

  6276.         self.assertHistory([])

  6277.         self.assertEqual(A.reported, 1)

  6278. 

  6279.     def test_fix_check_with_running(self):

  6280.         "Test that we can fix a broken check pipeline (do run the job)"

  6281.         in_repo_conf = textwrap.dedent(

  6282.             """

  6283.             - job:

  6284.                 name: existing-files

  6285.                 files:

  6286.                   - README.txt

  6287. 

  6288.             - project-template:

  6289.                 name: files-template

  6290.                 check:

  6291.                   jobs:

  6292.                     - existing-files

  6293.             """)

  6294. 

  6295.         # When the config is broken, we don't override any files

  6296.         # matchers since we don't have a valid basis.  Since this

  6297.         # does update README.txt, the job should run.

  6298.         file_dict = {'zuul.d/template.yaml': in_repo_conf,

  6299.                      'README.txt': ''}

  6300.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',

  6301.                                            files=file_dict)

  6302.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6303.         self.waitUntilSettled()

  6304.         self.assertHistory([

  6305.             dict(name='existing-files', result='SUCCESS', changes='1,1'),

  6306.         ])

  6307.         self.assertEqual(A.reported, 1)

  6308. 

  6309. 

  6310. class TestJobUpdateFileMatcher(ZuulTestCase):

  6311.     tenant_config_file = 'config/job-update/main.yaml'

  6312. 

  6313.     def test_matchers(self):

  6314.         "Test matchers work as expected with no change"

  6315.         file_dict = {'README.txt': ''}

  6316.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',

  6317.                                            files=file_dict)

  6318.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6319.         self.waitUntilSettled()

  6320. 

  6321.         file_dict = {'something_else': ''}

  6322.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',

  6323.                                            files=file_dict)

  6324.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  6325.         self.waitUntilSettled()

  6326. 

  6327.         self.assertHistory([

  6328.             dict(name='existing-files', result='SUCCESS', changes='1,1'),

  6329.             dict(name='existing-irr', result='SUCCESS', changes='2,1'),

  6330.         ])

  6331. 

  6332.     def test_job_update(self):

  6333.         "Test matchers are overridden with a config update"

  6334.         in_repo_conf = textwrap.dedent(

  6335.             """

  6336.             - job:

  6337.                 name: existing-files

  6338.                 tags: foo

  6339.             - job:

  6340.                 name: existing-irr

  6341.                 tags: foo

  6342.             """)

  6343. 

  6344.         file_dict = {'zuul.d/new.yaml': in_repo_conf}

  6345.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',

  6346.                                            files=file_dict)

  6347.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6348.         self.waitUntilSettled()

  6349. 

  6350.         self.assertHistory([

  6351.             dict(name='existing-files', result='SUCCESS', changes='1,1'),

  6352.             dict(name='existing-irr', result='SUCCESS', changes='1,1'),

  6353.         ], ordered=False)

  6354. 

  6355.     def test_job_update_files(self):

  6356.         "Test that changes to file matchers themselves don't run jobs"

  6357.         # Normally we want to ignore file matchers and run jobs if the

  6358.         # job config changes, but if the only thing about the job

  6359.         # config that changes *is* the file matchers, then we don't

  6360.         # want to run it.

  6361.         in_repo_conf = textwrap.dedent(

  6362.             """

  6363.             - job:

  6364.                 name: existing-files

  6365.                 files: 'doesnotexist'

  6366.             - job:

  6367.                 name: existing-irr

  6368.                 irrelevant-files:

  6369.                   - README

  6370.                   - ^zuul.d/.*$

  6371.                   - newthing

  6372.             """)

  6373. 

  6374.         file_dict = {'zuul.d/new.yaml': in_repo_conf}

  6375.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',

  6376.                                            files=file_dict)

  6377.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6378.         self.waitUntilSettled()

  6379. 

  6380.         self.assertHistory([])

  6381. 

  6382.     def test_new_job(self):

  6383.         "Test matchers are overridden when creating a new job"

  6384.         in_repo_conf = textwrap.dedent(

  6385.             """

  6386.             - job:

  6387.                 name: new-files

  6388.                 parent: existing-files

  6389. 

  6390.             - project:

  6391.                 check:

  6392.                   jobs:

  6393.                     - new-files

  6394.             """)

  6395. 

  6396.         file_dict = {'zuul.d/new.yaml': in_repo_conf}

  6397.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',

  6398.                                            files=file_dict)

  6399.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6400.         self.waitUntilSettled()

  6401. 

  6402.         self.assertHistory([

  6403.             dict(name='new-files', result='SUCCESS', changes='1,1'),

  6404.         ])

  6405. 

  6406.     def test_patch_series(self):

  6407.         "Test that we diff to the nearest layout in a patch series"

  6408.         in_repo_conf = textwrap.dedent(

  6409.             """

  6410.             - job:

  6411.                 name: new-files1

  6412.                 parent: existing-files

  6413. 

  6414.             - project:

  6415.                 check:

  6416.                   jobs:

  6417.                     - new-files1

  6418.             """)

  6419. 

  6420.         file_dict = {'zuul.d/new1.yaml': in_repo_conf}

  6421.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',

  6422.                                            files=file_dict)

  6423. 

  6424.         in_repo_conf = textwrap.dedent(

  6425.             """

  6426.             - job:

  6427.                 name: new-files2

  6428.                 parent: existing-files

  6429. 

  6430.             - project:

  6431.                 check:

  6432.                   jobs:

  6433.                     - new-files2

  6434.             """)

  6435. 

  6436.         file_dict = {'zuul.d/new2.yaml': in_repo_conf}

  6437.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',

  6438.                                            files=file_dict)

  6439.         B.setDependsOn(A, 1)

  6440.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  6441.         self.waitUntilSettled()

  6442. 

  6443.         self.assertHistory([

  6444.             dict(name='new-files2', result='SUCCESS', changes='1,1 2,1'),

  6445.         ])

  6446. 

  6447.     def test_disable_match(self):

  6448.         "Test matchers are not overridden if we say so"

  6449.         in_repo_conf = textwrap.dedent(

  6450.             """

  6451.             - job:

  6452.                 name: new-files

  6453.                 parent: existing-files

  6454.                 match-on-config-updates: false

  6455. 

  6456.             - project:

  6457.                 check:

  6458.                   jobs:

  6459.                     - new-files

  6460.             """)

  6461. 

  6462.         file_dict = {'zuul.d/new.yaml': in_repo_conf}

  6463.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',

  6464.                                            files=file_dict)

  6465.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6466.         self.waitUntilSettled()

  6467.         self.assertHistory([])

  6468. 

  6469. 

  6470. class TestAmbiguousProjectNames(ZuulTestCase):

  6471.     config_file = 'zuul-connections-multiple-gerrits.conf'

  6472.     tenant_config_file = 'config/ambiguous-names/main.yaml'

  6473. 

  6474.     def test_client_enqueue_canonical(self):

  6475.         "Test that the RPC client can enqueue a change using canonical name"

  6476.         A = self.fake_review_gerrit.addFakeChange('org/project', 'master', 'A')

  6477.         A.addApproval('Code-Review', 2)

  6478.         A.addApproval('Approved', 1)

  6479. 

  6480.         client = zuul.rpcclient.RPCClient('127.0.0.1',

  6481.                                           self.gearman_server.port)

  6482.         self.addCleanup(client.shutdown)

  6483.         r = client.enqueue(tenant='tenant-one',

  6484.                            pipeline='check',

  6485.                            project='review.example.com/org/project',

  6486.                            trigger=None,

  6487.                            change='1,1')

  6488.         self.waitUntilSettled()

  6489.         self.assertEqual(self.getJobFromHistory('project-merge').result,

  6490.                          'SUCCESS')

  6491.         self.assertEqual(r, True)

  6492. 

  6493. 

  6494. class TestExecutor(ZuulTestCase):

  6495.     tenant_config_file = 'config/single-tenant/main.yaml'

  6496. 

  6497.     def assertFinalState(self):

  6498.         # In this test, we expect to shut down in a non-final state,

  6499.         # so skip these checks.

  6500.         pass

  6501. 

  6502.     def assertCleanShutdown(self):

  6503.         self.log.debug("Assert clean shutdown")

  6504. 

  6505.         # After shutdown, make sure no jobs are running

  6506.         self.assertEqual({}, self.executor_server.job_workers)

  6507. 

  6508.         # Make sure that git.Repo objects have been garbage collected.

  6509.         gc.disable()

  6510.         try:

  6511.             gc.collect()

  6512.             for obj in gc.get_objects():

  6513.                 if isinstance(obj, git.Repo):

  6514.                     self.log.debug("Leaked git repo object: %s" % repr(obj))

  6515.             gc.enable()

  6516.         finally:

  6517.             gc.enable()

  6518. 

  6519.     def test_executor_shutdown(self):

  6520.         "Test that the executor can shut down with jobs running"

  6521. 

  6522.         self.executor_server.hold_jobs_in_build = True

  6523.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  6524.         A.addApproval('Code-Review', 2)

  6525.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  6526.         self.waitUntilSettled()

  6527. 

  6528. 

  6529. class TestDependencyGraph(ZuulTestCase):

  6530.     tenant_config_file = 'config/dependency-graph/main.yaml'

  6531. 

  6532.     def test_dependeny_graph_dispatch_jobs_once(self):

  6533.         "Test a job in a dependency graph is queued only once"

  6534.         # Job dependencies, starting with A

  6535.         #     A

  6536.         #    / \

  6537.         #   B   C

  6538.         #  / \ / \

  6539.         # D   F   E

  6540.         #     |

  6541.         #     G

  6542. 

  6543.         self.executor_server.hold_jobs_in_build = True

  6544.         change = self.fake_gerrit.addFakeChange(

  6545.             'org/project', 'master', 'change')

  6546.         change.addApproval('Code-Review', 2)

  6547.         self.fake_gerrit.addEvent(change.addApproval('Approved', 1))

  6548. 

  6549.         self.waitUntilSettled()

  6550.         self.assertEqual([b.name for b in self.builds], ['A'])

  6551. 

  6552.         self.executor_server.release('A')

  6553.         self.waitUntilSettled()

  6554.         self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])

  6555. 

  6556.         self.executor_server.release('B')

  6557.         self.waitUntilSettled()

  6558.         self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])

  6559. 

  6560.         self.executor_server.release('D')

  6561.         self.waitUntilSettled()

  6562.         self.assertEqual([b.name for b in self.builds], ['C'])

  6563. 

  6564.         self.executor_server.release('C')

  6565.         self.waitUntilSettled()

  6566.         self.assertEqual(sorted(b.name for b in self.builds), ['E', 'F'])

  6567. 

  6568.         self.executor_server.release('F')

  6569.         self.waitUntilSettled()

  6570.         self.assertEqual(sorted(b.name for b in self.builds), ['E', 'G'])

  6571. 

  6572.         self.executor_server.release('G')

  6573.         self.waitUntilSettled()

  6574.         self.assertEqual([b.name for b in self.builds], ['E'])

  6575. 

  6576.         self.executor_server.release('E')

  6577.         self.waitUntilSettled()

  6578.         self.assertEqual(len(self.builds), 0)

  6579. 

  6580.         self.executor_server.hold_jobs_in_build = False

  6581.         self.executor_server.release()

  6582.         self.waitUntilSettled()

  6583. 

  6584.         self.assertEqual(len(self.builds), 0)

  6585.         self.assertEqual(len(self.history), 7)

  6586. 

  6587.         self.assertEqual(change.data['status'], 'MERGED')

  6588.         self.assertEqual(change.reported, 2)

  6589. 

  6590.     def test_jobs_launched_only_if_all_dependencies_are_successful(self):

  6591.         "Test that a job waits till all dependencies are successful"

  6592.         # Job dependencies, starting with A

  6593.         #     A

  6594.         #    / \

  6595.         #   B   C*

  6596.         #  / \ / \

  6597.         # D   F   E

  6598.         #     |

  6599.         #     G

  6600. 

  6601.         self.executor_server.hold_jobs_in_build = True

  6602.         change = self.fake_gerrit.addFakeChange(

  6603.             'org/project', 'master', 'change')

  6604.         change.addApproval('Code-Review', 2)

  6605. 

  6606.         self.executor_server.failJob('C', change)

  6607. 

  6608.         self.fake_gerrit.addEvent(change.addApproval('Approved', 1))

  6609. 

  6610.         self.waitUntilSettled()

  6611.         self.assertEqual([b.name for b in self.builds], ['A'])

  6612. 

  6613.         self.executor_server.release('A')

  6614.         self.waitUntilSettled()

  6615.         self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])

  6616. 

  6617.         self.executor_server.release('B')

  6618.         self.waitUntilSettled()

  6619.         self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])

  6620. 

  6621.         self.executor_server.release('D')

  6622.         self.waitUntilSettled()

  6623.         self.assertEqual([b.name for b in self.builds], ['C'])

  6624. 

  6625.         self.executor_server.release('C')

  6626.         self.waitUntilSettled()

  6627.         self.assertEqual(len(self.builds), 0)

  6628. 

  6629.         self.executor_server.hold_jobs_in_build = False

  6630.         self.executor_server.release()

  6631.         self.waitUntilSettled()

  6632. 

  6633.         self.assertEqual(len(self.builds), 0)

  6634.         self.assertEqual(len(self.history), 4)

  6635. 

  6636.         self.assertEqual(change.data['status'], 'NEW')

  6637.         self.assertEqual(change.reported, 2)

  6638. 

  6639.     @simple_layout('layouts/soft-dependencies-error.yaml')

  6640.     def test_soft_dependencies_error(self):

  6641.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  6642.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6643.         self.waitUntilSettled()

  6644.         self.assertHistory([])

  6645.         self.assertEqual(len(A.messages), 1)

  6646.         self.assertTrue('Job project-merge not defined' in A.messages[0])

  6647.         print(A.messages)

  6648. 

  6649.     @simple_layout('layouts/soft-dependencies.yaml')

  6650.     def test_soft_dependencies(self):

  6651.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  6652.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6653.         self.waitUntilSettled()

  6654.         self.assertHistory([

  6655.             dict(name='deploy', result='SUCCESS', changes='1,1'),

  6656.         ], ordered=False)

  6657. 

  6658.     @simple_layout('layouts/soft-dependencies.yaml')

  6659.     def test_soft_dependencies_failure(self):

  6660.         file_dict = {'main.c': 'test'}

  6661.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',

  6662.                                            files=file_dict)

  6663.         self.executor_server.failJob('build', A)

  6664.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6665.         self.waitUntilSettled()

  6666.         self.assertHistory([

  6667.             dict(name='build', result='FAILURE', changes='1,1'),

  6668.         ], ordered=False)

  6669.         self.assertIn('SKIPPED', A.messages[0])

  6670. 

  6671. 

  6672. class TestDuplicatePipeline(ZuulTestCase):

  6673.     tenant_config_file = 'config/duplicate-pipeline/main.yaml'

  6674. 

  6675.     def test_duplicate_pipelines(self):

  6676.         "Test that a change matching multiple pipelines works"

  6677. 

  6678.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  6679.         self.fake_gerrit.addEvent(A.getChangeRestoredEvent())

  6680.         self.waitUntilSettled()

  6681. 

  6682.         self.assertHistory([

  6683.             dict(name='project-test1', result='SUCCESS', changes='1,1',

  6684.                  pipeline='dup1'),

  6685.             dict(name='project-test1', result='SUCCESS', changes='1,1',

  6686.                  pipeline='dup2'),

  6687.         ], ordered=False)

  6688. 

  6689.         self.assertEqual(len(A.messages), 2)

  6690. 

  6691.         if 'dup1' in A.messages[0]:

  6692.             self.assertIn('dup1', A.messages[0])

  6693.             self.assertNotIn('dup2', A.messages[0])

  6694.             self.assertIn('project-test1', A.messages[0])

  6695.             self.assertIn('dup2', A.messages[1])

  6696.             self.assertNotIn('dup1', A.messages[1])

  6697.             self.assertIn('project-test1', A.messages[1])

  6698.         else:

  6699.             self.assertIn('dup1', A.messages[1])

  6700.             self.assertNotIn('dup2', A.messages[1])

  6701.             self.assertIn('project-test1', A.messages[1])

  6702.             self.assertIn('dup2', A.messages[0])

  6703.             self.assertNotIn('dup1', A.messages[0])

  6704.             self.assertIn('project-test1', A.messages[0])

  6705. 

  6706. 

  6707. class TestSchedulerRegexProject(ZuulTestCase):

  6708.     tenant_config_file = 'config/regex-project/main.yaml'

  6709. 

  6710.     def test_regex_project(self):

  6711.         "Test that changes are tested in parallel and merged in series"

  6712. 

  6713.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  6714.         B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')

  6715.         C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')

  6716. 

  6717.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6718.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  6719.         self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))

  6720. 

  6721.         self.waitUntilSettled()

  6722. 

  6723.         # We expect the following builds:

  6724.         #  - 1 for org/project

  6725.         #  - 3 for org/project1

  6726.         #  - 3 for org/project2

  6727.         self.assertEqual(len(self.history), 7)

  6728.         self.assertEqual(A.reported, 1)

  6729.         self.assertEqual(B.reported, 1)

  6730.         self.assertEqual(C.reported, 1)

  6731. 

  6732.         self.assertHistory([

  6733.             dict(name='project-test', result='SUCCESS', changes='1,1'),

  6734.             dict(name='project-test1', result='SUCCESS', changes='2,1'),

  6735.             dict(name='project-common-test', result='SUCCESS', changes='2,1'),

  6736.             dict(name='project-common-test-canonical', result='SUCCESS',

  6737.                  changes='2,1'),

  6738.             dict(name='project-test2', result='SUCCESS', changes='3,1'),

  6739.             dict(name='project-common-test', result='SUCCESS', changes='3,1'),

  6740.             dict(name='project-common-test-canonical', result='SUCCESS',

  6741.                  changes='3,1'),

  6742.         ], ordered=False)

  6743. 

  6744. 

  6745. class TestSchedulerTemplatedProject(ZuulTestCase):

  6746.     tenant_config_file = 'config/templated-project/main.yaml'

  6747. 

  6748.     def test_job_from_templates_executed(self):

  6749.         "Test whether a job generated via a template can be executed"

  6750. 

  6751.         A = self.fake_gerrit.addFakeChange(

  6752.             'org/templated-project', 'master', 'A')

  6753.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6754.         self.waitUntilSettled()

  6755. 

  6756.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  6757.                          'SUCCESS')

  6758.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  6759.                          'SUCCESS')

  6760. 

  6761.     def test_layered_templates(self):

  6762.         "Test whether a job generated via a template can be executed"

  6763. 

  6764.         A = self.fake_gerrit.addFakeChange(

  6765.             'org/layered-project', 'master', 'A')

  6766.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6767.         self.waitUntilSettled()

  6768. 

  6769.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  6770.                          'SUCCESS')

  6771.         self.assertEqual(self.getJobFromHistory('project-test2').result,

  6772.                          'SUCCESS')

  6773.         self.assertEqual(self.getJobFromHistory('layered-project-test3'

  6774.                                                 ).result, 'SUCCESS')

  6775.         self.assertEqual(self.getJobFromHistory('layered-project-test4'

  6776.                                                 ).result, 'SUCCESS')

  6777.         self.assertEqual(self.getJobFromHistory('layered-project-foo-test5'

  6778.                                                 ).result, 'SUCCESS')

  6779.         self.assertEqual(self.getJobFromHistory('project-test6').result,

  6780.                          'SUCCESS')

  6781. 

  6782.     def test_unimplied_branch_matchers(self):

  6783.         # This tests that there are no implied branch matchers added

  6784.         # to project templates in unbranched projects.

  6785.         self.create_branch('org/layered-project', 'stable')

  6786.         self.fake_gerrit.addEvent(

  6787.             self.fake_gerrit.getFakeBranchCreatedEvent(

  6788.                 'org/layered-project', 'stable'))

  6789.         self.waitUntilSettled()

  6790. 

  6791.         A = self.fake_gerrit.addFakeChange(

  6792.             'org/layered-project', 'stable', 'A')

  6793. 

  6794.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6795.         self.waitUntilSettled()

  6796. 

  6797.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  6798.                          'SUCCESS')

  6799.         self.log.info(

  6800.             self.getJobFromHistory('project-test1').

  6801.             parameters['zuul']['_inheritance_path'])

  6802. 

  6803.     def test_implied_branch_matchers(self):

  6804.         # This tests that there is an implied branch matcher when a

  6805.         # template is used on an in-repo project pipeline definition.

  6806.         self.create_branch('untrusted-config', 'stable')

  6807.         self.fake_gerrit.addEvent(

  6808.             self.fake_gerrit.getFakeBranchCreatedEvent(

  6809.                 'untrusted-config', 'stable'))

  6810.         self.waitUntilSettled()

  6811. 

  6812.         A = self.fake_gerrit.addFakeChange(

  6813.             'untrusted-config', 'stable', 'A')

  6814. 

  6815.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6816.         self.waitUntilSettled()

  6817. 

  6818.         self.assertEqual(self.getJobFromHistory('project-test1').result,

  6819.                          'SUCCESS')

  6820.         self.log.info(

  6821.             self.getJobFromHistory('project-test1').

  6822.             parameters['zuul']['_inheritance_path'])

  6823. 

  6824.         # Now create a new branch named stable-foo and change the project

  6825.         # pipeline

  6826.         self.create_branch('untrusted-config', 'stable-foo')

  6827.         self.fake_gerrit.addEvent(

  6828.             self.fake_gerrit.getFakeBranchCreatedEvent(

  6829.                 'untrusted-config', 'stable-foo'))

  6830.         self.waitUntilSettled()

  6831. 

  6832.         in_repo_conf = textwrap.dedent(

  6833.             """

  6834.             - project:

  6835.                 name: untrusted-config

  6836.                 templates:

  6837.                   - test-three-and-four

  6838.             """)

  6839.         file_dict = {'zuul.d/project.yaml': in_repo_conf}

  6840.         B = self.fake_gerrit.addFakeChange('untrusted-config', 'stable-foo',

  6841.                                            'B', files=file_dict)

  6842.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  6843.         self.waitUntilSettled()

  6844. 

  6845.         self.assertHistory([

  6846.             dict(name='project-test1', result='SUCCESS', changes='1,1'),

  6847.             dict(name='project-test2', result='SUCCESS', changes='1,1'),

  6848.             dict(name='layered-project-test3', result='SUCCESS',

  6849.                  changes='2,1'),

  6850.             dict(name='layered-project-test4', result='SUCCESS',

  6851.                  changes='2,1'),

  6852.         ], ordered=False)

  6853. 

  6854. 

  6855. class TestSchedulerSuccessURL(ZuulTestCase):

  6856.     tenant_config_file = 'config/success-url/main.yaml'

  6857. 

  6858.     def test_success_url(self):

  6859.         "Ensure bad build params are ignored"

  6860.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  6861.         self.init_repo('org/docs')

  6862. 

  6863.         A = self.fake_gerrit.addFakeChange('org/docs', 'master', 'A')

  6864.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  6865.         self.waitUntilSettled()

  6866. 

  6867.         # Both builds ran: docs-draft-test + docs-draft-test2

  6868.         self.assertEqual(len(self.history), 2)

  6869. 

  6870.         # Grab build id

  6871.         for build in self.history:

  6872.             if build.name == 'docs-draft-test':

  6873.                 uuid = build.uuid[:7]

  6874.             elif build.name == 'docs-draft-test2':

  6875.                 uuid_test2 = build.uuid

  6876. 

  6877.         # Two msgs: 'Starting...'  + results

  6878.         self.assertEqual(len(self.smtp_messages), 2)

  6879.         body = self.smtp_messages[1]['body'].splitlines()

  6880.         self.assertEqual('Build succeeded.', body[0])

  6881. 

  6882.         self.assertIn(

  6883.             '- docs-draft-test http://docs-draft.example.org/1/1/1/check/'

  6884.             'docs-draft-test/%s/publish-docs/' % uuid,

  6885.             body[2])

  6886. 

  6887.         # NOTE: This default URL is currently hard-coded in executor/server.py

  6888.         self.assertIn(

  6889.             '- docs-draft-test2 finger://{hostname}/{uuid}'.format(

  6890.                 hostname=self.executor_server.hostname,

  6891.                 uuid=uuid_test2),

  6892.             body[3])

  6893. 

  6894. 

  6895. class TestSchedulerMerges(ZuulTestCase):

  6896.     tenant_config_file = 'config/merges/main.yaml'

  6897. 

  6898.     def _test_project_merge_mode(self, mode):

  6899.         self.executor_server.keep_jobdir = False

  6900.         project = 'org/project-%s' % mode

  6901.         self.executor_server.hold_jobs_in_build = True

  6902.         A = self.fake_gerrit.addFakeChange(project, 'master', 'A')

  6903.         B = self.fake_gerrit.addFakeChange(project, 'master', 'B')

  6904.         C = self.fake_gerrit.addFakeChange(project, 'master', 'C')

  6905.         A.addApproval('Code-Review', 2)

  6906.         B.addApproval('Code-Review', 2)

  6907.         C.addApproval('Code-Review', 2)

  6908.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  6909.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  6910.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  6911.         self.waitUntilSettled()

  6912. 

  6913.         build = self.builds[-1]

  6914.         path = os.path.join(build.jobdir.src_root, 'review.example.com',

  6915.                             project)

  6916.         repo = git.Repo(path)

  6917.         repo_messages = [c.message.strip() for c in repo.iter_commits()]

  6918.         repo_messages.reverse()

  6919. 

  6920.         self.executor_server.hold_jobs_in_build = False

  6921.         self.executor_server.release()

  6922.         self.waitUntilSettled()

  6923. 

  6924.         return repo_messages

  6925. 

  6926.     def _test_merge(self, mode):

  6927.         us_path = 'file://' + os.path.join(

  6928.             self.upstream_root, 'org/project-%s' % mode)

  6929.         expected_messages = [

  6930.             'initial commit',

  6931.             'add content from fixture',

  6932.             # the intermediate commits order is nondeterministic

  6933.             "Merge commit 'refs/changes/1/2/1' of %s into HEAD" % us_path,

  6934.             "Merge commit 'refs/changes/1/3/1' of %s into HEAD" % us_path,

  6935.         ]

  6936.         result = self._test_project_merge_mode(mode)

  6937.         self.assertEqual(result[:2], expected_messages[:2])

  6938.         self.assertEqual(result[-2:], expected_messages[-2:])

  6939. 

  6940.     def test_project_merge_mode_merge(self):

  6941.         self._test_merge('merge')

  6942. 

  6943.     def test_project_merge_mode_merge_resolve(self):

  6944.         self._test_merge('merge-resolve')

  6945. 

  6946.     def test_project_merge_mode_cherrypick(self):

  6947.         expected_messages = [

  6948.             'initial commit',

  6949.             'add content from fixture',

  6950.             'A-1',

  6951.             'B-1',

  6952.             'C-1']

  6953.         result = self._test_project_merge_mode('cherry-pick')

  6954.         self.assertEqual(result, expected_messages)

  6955. 

  6956.     def test_merge_branch(self):

  6957.         "Test that the right commits are on alternate branches"

  6958.         self.create_branch('org/project-merge-branches', 'mp')

  6959.         self.fake_gerrit.addEvent(

  6960.             self.fake_gerrit.getFakeBranchCreatedEvent(

  6961.                 'org/project-merge-branches', 'mp'))

  6962.         self.waitUntilSettled()

  6963. 

  6964.         self.executor_server.hold_jobs_in_build = True

  6965.         A = self.fake_gerrit.addFakeChange(

  6966.             'org/project-merge-branches', 'mp', 'A')

  6967.         B = self.fake_gerrit.addFakeChange(

  6968.             'org/project-merge-branches', 'mp', 'B')

  6969.         C = self.fake_gerrit.addFakeChange(

  6970.             'org/project-merge-branches', 'mp', 'C')

  6971.         A.addApproval('Code-Review', 2)

  6972.         B.addApproval('Code-Review', 2)

  6973.         C.addApproval('Code-Review', 2)

  6974.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  6975.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  6976.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  6977.         self.waitUntilSettled()

  6978. 

  6979.         self.executor_server.release('.*-merge')

  6980.         self.waitUntilSettled()

  6981.         self.executor_server.release('.*-merge')

  6982.         self.waitUntilSettled()

  6983.         self.executor_server.release('.*-merge')

  6984.         self.waitUntilSettled()

  6985. 

  6986.         build = self.builds[-1]

  6987.         self.assertEqual(build.parameters['zuul']['branch'], 'mp')

  6988.         path = os.path.join(build.jobdir.src_root, 'review.example.com',

  6989.                             'org/project-merge-branches')

  6990.         repo = git.Repo(path)

  6991. 

  6992.         repo_messages = [c.message.strip() for c in repo.iter_commits()]

  6993.         repo_messages.reverse()

  6994.         correct_messages = [

  6995.             'initial commit',

  6996.             'add content from fixture',

  6997.             'mp commit',

  6998.             'A-1', 'B-1', 'C-1']

  6999.         self.assertEqual(repo_messages, correct_messages)

  7000. 

  7001.         self.executor_server.hold_jobs_in_build = False

  7002.         self.executor_server.release()

  7003.         self.waitUntilSettled()

  7004. 

  7005.     def test_merge_multi_branch(self):

  7006.         "Test that dependent changes on multiple branches are merged"

  7007.         self.create_branch('org/project-merge-branches', 'mp')

  7008.         self.fake_gerrit.addEvent(

  7009.             self.fake_gerrit.getFakeBranchCreatedEvent(

  7010.                 'org/project-merge-branches', 'mp'))

  7011.         self.waitUntilSettled()

  7012. 

  7013.         self.executor_server.hold_jobs_in_build = True

  7014.         A = self.fake_gerrit.addFakeChange(

  7015.             'org/project-merge-branches', 'master', 'A')

  7016.         B = self.fake_gerrit.addFakeChange(

  7017.             'org/project-merge-branches', 'mp', 'B')

  7018.         C = self.fake_gerrit.addFakeChange(

  7019.             'org/project-merge-branches', 'master', 'C')

  7020.         A.addApproval('Code-Review', 2)

  7021.         B.addApproval('Code-Review', 2)

  7022.         C.addApproval('Code-Review', 2)

  7023.         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))

  7024.         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))

  7025.         self.fake_gerrit.addEvent(C.addApproval('Approved', 1))

  7026.         self.waitUntilSettled()

  7027. 

  7028.         job_A = None

  7029.         for job in self.builds:

  7030.             if 'project-merge' in job.name:

  7031.                 job_A = job

  7032. 

  7033.         path = os.path.join(job_A.jobdir.src_root, 'review.example.com',

  7034.                             'org/project-merge-branches')

  7035.         repo = git.Repo(path)

  7036.         repo_messages = [c.message.strip()

  7037.                          for c in repo.iter_commits()]

  7038.         repo_messages.reverse()

  7039.         correct_messages = [

  7040.             'initial commit', 'add content from fixture', 'A-1']

  7041.         self.assertEqual(repo_messages, correct_messages)

  7042. 

  7043.         self.executor_server.release('.*-merge')

  7044.         self.waitUntilSettled()

  7045. 

  7046.         job_B = None

  7047.         for job in self.builds:

  7048.             if 'project-merge' in job.name:

  7049.                 job_B = job

  7050. 

  7051.         path = os.path.join(job_B.jobdir.src_root, 'review.example.com',

  7052.                             'org/project-merge-branches')

  7053.         repo = git.Repo(path)

  7054.         repo_messages = [c.message.strip() for c in repo.iter_commits()]

  7055.         repo_messages.reverse()

  7056.         correct_messages = [

  7057.             'initial commit', 'add content from fixture', 'mp commit', 'B-1']

  7058.         self.assertEqual(repo_messages, correct_messages)

  7059. 

  7060.         self.executor_server.release('.*-merge')

  7061.         self.waitUntilSettled()

  7062. 

  7063.         job_C = None

  7064.         for job in self.builds:

  7065.             if 'project-merge' in job.name:

  7066.                 job_C = job

  7067. 

  7068.         path = os.path.join(job_C.jobdir.src_root, 'review.example.com',

  7069.                             'org/project-merge-branches')

  7070.         repo = git.Repo(path)

  7071.         repo_messages = [c.message.strip() for c in repo.iter_commits()]

  7072. 

  7073.         repo_messages.reverse()

  7074.         correct_messages = [

  7075.             'initial commit', 'add content from fixture',

  7076.             'A-1', 'C-1']

  7077.         # Ensure the right commits are in the history for this ref

  7078.         self.assertEqual(repo_messages, correct_messages)

  7079. 

  7080.         self.executor_server.hold_jobs_in_build = False

  7081.         self.executor_server.release()

  7082.         self.waitUntilSettled()

  7083. 

  7084. 

  7085. class TestSemaphore(ZuulTestCase):

  7086.     tenant_config_file = 'config/semaphore/main.yaml'

  7087. 

  7088.     def test_semaphore_one(self):

  7089.         "Test semaphores with max=1 (mutex)"

  7090.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7091. 

  7092.         self.executor_server.hold_jobs_in_build = True

  7093. 

  7094.         # Pause nodepool so we can check the ordering of getting the nodes

  7095.         # and aquiring the semaphore.

  7096.         self.fake_nodepool.paused = True

  7097. 

  7098.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  7099.         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')

  7100.         self.assertFalse('test-semaphore' in

  7101.                          tenant.semaphore_handler.semaphores)

  7102. 

  7103.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7104.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  7105.         self.waitUntilSettled()

  7106. 

  7107.         # By default we first lock the semaphore and then get the nodes

  7108.         # so at this point the semaphore needs to be aquired.

  7109.         self.assertTrue('test-semaphore' in

  7110.                         tenant.semaphore_handler.semaphores)

  7111.         self.fake_nodepool.paused = False

  7112.         self.waitUntilSettled()

  7113. 

  7114.         self.assertEqual(len(self.builds), 3)

  7115.         self.assertEqual(self.builds[0].name, 'project-test1')

  7116.         self.assertEqual(self.builds[1].name, 'semaphore-one-test1')

  7117.         self.assertEqual(self.builds[2].name, 'project-test1')

  7118. 

  7119.         self.executor_server.release('semaphore-one-test1')

  7120.         self.waitUntilSettled()

  7121. 

  7122.         self.assertEqual(len(self.builds), 3)

  7123.         self.assertEqual(self.builds[0].name, 'project-test1')

  7124.         self.assertEqual(self.builds[1].name, 'project-test1')

  7125.         self.assertEqual(self.builds[2].name, 'semaphore-one-test2')

  7126.         self.assertTrue('test-semaphore' in

  7127.                         tenant.semaphore_handler.semaphores)

  7128. 

  7129.         self.executor_server.release('semaphore-one-test2')

  7130.         self.waitUntilSettled()

  7131. 

  7132.         self.assertEqual(len(self.builds), 3)

  7133.         self.assertEqual(self.builds[0].name, 'project-test1')

  7134.         self.assertEqual(self.builds[1].name, 'project-test1')

  7135.         self.assertEqual(self.builds[2].name, 'semaphore-one-test1')

  7136.         self.assertTrue('test-semaphore' in

  7137.                         tenant.semaphore_handler.semaphores)

  7138. 

  7139.         self.executor_server.release('semaphore-one-test1')

  7140.         self.waitUntilSettled()

  7141. 

  7142.         self.assertEqual(len(self.builds), 3)

  7143.         self.assertEqual(self.builds[0].name, 'project-test1')

  7144.         self.assertEqual(self.builds[1].name, 'project-test1')

  7145.         self.assertEqual(self.builds[2].name, 'semaphore-one-test2')

  7146.         self.assertTrue('test-semaphore' in

  7147.                         tenant.semaphore_handler.semaphores)

  7148. 

  7149.         self.executor_server.release('semaphore-one-test2')

  7150.         self.waitUntilSettled()

  7151. 

  7152.         self.assertEqual(len(self.builds), 2)

  7153.         self.assertEqual(self.builds[0].name, 'project-test1')

  7154.         self.assertEqual(self.builds[1].name, 'project-test1')

  7155.         self.assertFalse('test-semaphore' in

  7156.                          tenant.semaphore_handler.semaphores)

  7157. 

  7158.         self.executor_server.hold_jobs_in_build = False

  7159.         self.executor_server.release()

  7160. 

  7161.         self.waitUntilSettled()

  7162.         self.assertEqual(len(self.builds), 0)

  7163. 

  7164.         self.assertEqual(A.reported, 1)

  7165.         self.assertEqual(B.reported, 1)

  7166.         self.assertFalse('test-semaphore' in

  7167.                          tenant.semaphore_handler.semaphores)

  7168. 

  7169.     def test_semaphore_two(self):

  7170.         "Test semaphores with max>1"

  7171.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7172. 

  7173.         self.executor_server.hold_jobs_in_build = True

  7174.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  7175.         B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')

  7176.         self.assertFalse('test-semaphore-two' in

  7177.                          tenant.semaphore_handler.semaphores)

  7178. 

  7179.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7180.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  7181.         self.waitUntilSettled()

  7182. 

  7183.         self.assertEqual(len(self.builds), 4)

  7184.         self.assertEqual(self.builds[0].name, 'project-test1')

  7185.         self.assertEqual(self.builds[1].name, 'semaphore-two-test1')

  7186.         self.assertEqual(self.builds[2].name, 'semaphore-two-test2')

  7187.         self.assertEqual(self.builds[3].name, 'project-test1')

  7188.         self.assertTrue('test-semaphore-two' in

  7189.                         tenant.semaphore_handler.semaphores)

  7190.         self.assertEqual(len(tenant.semaphore_handler.semaphores.get(

  7191.             'test-semaphore-two', [])), 2)

  7192. 

  7193.         self.executor_server.release('semaphore-two-test1')

  7194.         self.waitUntilSettled()

  7195. 

  7196.         self.assertEqual(len(self.builds), 4)

  7197.         self.assertEqual(self.builds[0].name, 'project-test1')

  7198.         self.assertEqual(self.builds[1].name, 'semaphore-two-test2')

  7199.         self.assertEqual(self.builds[2].name, 'project-test1')

  7200.         self.assertEqual(self.builds[3].name, 'semaphore-two-test1')

  7201.         self.assertTrue('test-semaphore-two' in

  7202.                         tenant.semaphore_handler.semaphores)

  7203.         self.assertEqual(len(tenant.semaphore_handler.semaphores.get(

  7204.             'test-semaphore-two', [])), 2)

  7205. 

  7206.         self.executor_server.release('semaphore-two-test2')

  7207.         self.waitUntilSettled()

  7208. 

  7209.         self.assertEqual(len(self.builds), 4)

  7210.         self.assertEqual(self.builds[0].name, 'project-test1')

  7211.         self.assertEqual(self.builds[1].name, 'project-test1')

  7212.         self.assertEqual(self.builds[2].name, 'semaphore-two-test1')

  7213.         self.assertEqual(self.builds[3].name, 'semaphore-two-test2')

  7214.         self.assertTrue('test-semaphore-two' in

  7215.                         tenant.semaphore_handler.semaphores)

  7216.         self.assertEqual(len(tenant.semaphore_handler.semaphores.get(

  7217.             'test-semaphore-two', [])), 2)

  7218. 

  7219.         self.executor_server.release('semaphore-two-test1')

  7220.         self.waitUntilSettled()

  7221. 

  7222.         self.assertEqual(len(self.builds), 3)

  7223.         self.assertEqual(self.builds[0].name, 'project-test1')

  7224.         self.assertEqual(self.builds[1].name, 'project-test1')

  7225.         self.assertEqual(self.builds[2].name, 'semaphore-two-test2')

  7226.         self.assertTrue('test-semaphore-two' in

  7227.                         tenant.semaphore_handler.semaphores)

  7228.         self.assertEqual(len(tenant.semaphore_handler.semaphores.get(

  7229.             'test-semaphore-two', [])), 1)

  7230. 

  7231.         self.executor_server.release('semaphore-two-test2')

  7232.         self.waitUntilSettled()

  7233. 

  7234.         self.assertEqual(len(self.builds), 2)

  7235.         self.assertEqual(self.builds[0].name, 'project-test1')

  7236.         self.assertEqual(self.builds[1].name, 'project-test1')

  7237.         self.assertFalse('test-semaphore-two' in

  7238.                          tenant.semaphore_handler.semaphores)

  7239. 

  7240.         self.executor_server.hold_jobs_in_build = False

  7241.         self.executor_server.release()

  7242. 

  7243.         self.waitUntilSettled()

  7244.         self.assertEqual(len(self.builds), 0)

  7245. 

  7246.         self.assertEqual(A.reported, 1)

  7247.         self.assertEqual(B.reported, 1)

  7248. 

  7249.     def test_semaphore_node_failure(self):

  7250.         "Test semaphore and node failure"

  7251.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7252. 

  7253.         # Pause nodepool so we can fail the node request later

  7254.         self.fake_nodepool.pause()

  7255. 

  7256.         A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')

  7257.         self.assertFalse('test-semaphore' in

  7258.                          tenant.semaphore_handler.semaphores)

  7259.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7260.         self.waitUntilSettled()

  7261. 

  7262.         # By default we first lock the semaphore and then get the nodes

  7263.         # so at this point the semaphore needs to be aquired.

  7264.         self.assertTrue('test-semaphore' in

  7265.                         tenant.semaphore_handler.semaphores)

  7266. 

  7267.         # Fail the node request and unpause

  7268.         req = self.fake_nodepool.getNodeRequests()[0]

  7269.         self.fake_nodepool.addFailRequest(req)

  7270.         self.fake_nodepool.unpause()

  7271.         self.waitUntilSettled()

  7272. 

  7273.         # At this point the job that holds the semaphore failed with

  7274.         # node_failure and the semaphore must be released.

  7275.         self.assertFalse('test-semaphore' in

  7276.                          tenant.semaphore_handler.semaphores)

  7277.         self.assertEquals(1, A.reported)

  7278.         self.assertIn('semaphore-one-test3 semaphore-one-test3 : NODE_FAILURE',

  7279.                       A.messages[0])

  7280. 

  7281.     def test_semaphore_resources_first(self):

  7282.         "Test semaphores with max=1 (mutex) and get resources first"

  7283.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7284. 

  7285.         self.executor_server.hold_jobs_in_build = True

  7286. 

  7287.         # Pause nodepool so we can check the ordering of getting the nodes

  7288.         # and aquiring the semaphore.

  7289.         self.fake_nodepool.paused = True

  7290. 

  7291.         A = self.fake_gerrit.addFakeChange('org/project3', 'master', 'A')

  7292.         B = self.fake_gerrit.addFakeChange('org/project3', 'master', 'B')

  7293.         self.assertFalse('test-semaphore' in

  7294.                          tenant.semaphore_handler.semaphores)

  7295. 

  7296.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7297.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  7298.         self.waitUntilSettled()

  7299. 

  7300.         # Here we first get the resources and then lock the semaphore

  7301.         # so at this point the semaphore should not be aquired.

  7302.         self.assertFalse('test-semaphore' in

  7303.                          tenant.semaphore_handler.semaphores)

  7304.         self.fake_nodepool.paused = False

  7305.         self.waitUntilSettled()

  7306. 

  7307.         self.assertEqual(len(self.builds), 3)

  7308.         self.assertEqual(self.builds[0].name, 'project-test1')

  7309.         self.assertEqual(self.builds[1].name,

  7310.                          'semaphore-one-test1-resources-first')

  7311.         self.assertEqual(self.builds[2].name, 'project-test1')

  7312. 

  7313.         self.executor_server.release('semaphore-one-test1')

  7314.         self.waitUntilSettled()

  7315. 

  7316.         self.assertEqual(len(self.builds), 3)

  7317.         self.assertEqual(self.builds[0].name, 'project-test1')

  7318.         self.assertEqual(self.builds[1].name, 'project-test1')

  7319.         self.assertEqual(self.builds[2].name,

  7320.                          'semaphore-one-test2-resources-first')

  7321.         self.assertTrue('test-semaphore' in

  7322.                         tenant.semaphore_handler.semaphores)

  7323. 

  7324.         self.executor_server.hold_jobs_in_build = False

  7325.         self.executor_server.release()

  7326.         self.waitUntilSettled()

  7327. 

  7328.     def test_semaphore_resources_first_node_failure(self):

  7329.         "Test semaphore and node failure"

  7330.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7331. 

  7332.         # Pause nodepool so we can fail the node request later

  7333.         self.fake_nodepool.pause()

  7334. 

  7335.         A = self.fake_gerrit.addFakeChange('org/project4', 'master', 'A')

  7336.         self.assertFalse('test-semaphore' in

  7337.                          tenant.semaphore_handler.semaphores)

  7338.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7339.         self.waitUntilSettled()

  7340. 

  7341.         # With resources first we first get the nodes so at this point the

  7342.         # semaphore must not be aquired.

  7343.         self.assertFalse('test-semaphore' in

  7344.                          tenant.semaphore_handler.semaphores)

  7345. 

  7346.         # Fail the node request and unpause

  7347.         req = self.fake_nodepool.getNodeRequests()[0]

  7348.         self.fake_nodepool.addFailRequest(req)

  7349.         self.fake_nodepool.unpause()

  7350.         self.waitUntilSettled()

  7351. 

  7352.         # At this point the job should never have acuired a semaphore so check

  7353.         # that it still has not locked a semaphore.

  7354.         self.assertFalse('test-semaphore' in

  7355.                          tenant.semaphore_handler.semaphores)

  7356.         self.assertEquals(1, A.reported)

  7357.         self.assertIn('semaphore-one-test1-resources-first : NODE_FAILURE',

  7358.                       A.messages[0])

  7359. 

  7360.     def test_semaphore_zk_error(self):

  7361.         "Test semaphore release with zk error"

  7362.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7363. 

  7364.         A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')

  7365.         self.assertFalse('test-semaphore' in

  7366.                          tenant.semaphore_handler.semaphores)

  7367. 

  7368.         # Simulate a single zk error in useNodeSet

  7369.         orig_useNodeSet = self.scheds.first.sched.nodepool.useNodeSet

  7370. 

  7371.         def broken_use_nodeset(nodeset, build_set=None, event=None):

  7372.             # restore original useNodeSet

  7373.             self.scheds.first.sched.nodepool.useNodeSet = orig_useNodeSet

  7374.             raise NoNodeError()

  7375. 

  7376.         self.scheds.first.sched.nodepool.useNodeSet = broken_use_nodeset

  7377. 

  7378.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7379.         self.waitUntilSettled()

  7380. 

  7381.         # The semaphore should be released

  7382.         self.assertFalse('test-semaphore' in

  7383.                          tenant.semaphore_handler.semaphores)

  7384. 

  7385.         # cleanup the queue

  7386.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7387.         self.waitUntilSettled()

  7388. 

  7389.     def test_semaphore_abandon(self):

  7390.         "Test abandon with job semaphores"

  7391.         self.executor_server.hold_jobs_in_build = True

  7392.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7393.         check_pipeline = tenant.layout.pipelines['check']

  7394. 

  7395.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  7396.         self.assertFalse('test-semaphore' in

  7397.                          tenant.semaphore_handler.semaphores)

  7398. 

  7399.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7400.         self.waitUntilSettled()

  7401. 

  7402.         self.assertTrue('test-semaphore' in

  7403.                         tenant.semaphore_handler.semaphores)

  7404. 

  7405.         self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())

  7406.         self.waitUntilSettled()

  7407. 

  7408.         # The check pipeline should be empty

  7409.         items = check_pipeline.getAllItems()

  7410.         self.assertEqual(len(items), 0)

  7411. 

  7412.         # The semaphore should be released

  7413.         self.assertFalse('test-semaphore' in

  7414.                          tenant.semaphore_handler.semaphores)

  7415. 

  7416.         self.executor_server.hold_jobs_in_build = False

  7417.         self.executor_server.release()

  7418.         self.waitUntilSettled()

  7419. 

  7420.     def test_semaphore_abandon_pending_node_request(self):

  7421.         "Test abandon with job semaphores and pending node request"

  7422.         self.executor_server.hold_jobs_in_build = True

  7423. 

  7424.         # Pause nodepool so we can check the ordering of getting the nodes

  7425.         # and aquiring the semaphore.

  7426.         self.fake_nodepool.paused = True

  7427. 

  7428.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7429.         check_pipeline = tenant.layout.pipelines['check']

  7430. 

  7431.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  7432.         self.assertFalse('test-semaphore' in

  7433.                          tenant.semaphore_handler.semaphores)

  7434. 

  7435.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7436.         self.waitUntilSettled()

  7437. 

  7438.         self.assertTrue('test-semaphore' in

  7439.                         tenant.semaphore_handler.semaphores)

  7440. 

  7441.         self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())

  7442.         self.waitUntilSettled()

  7443. 

  7444.         # The check pipeline should be empty

  7445.         items = check_pipeline.getAllItems()

  7446.         self.assertEqual(len(items), 0)

  7447. 

  7448.         # The semaphore should be released

  7449.         self.assertFalse('test-semaphore' in

  7450.                          tenant.semaphore_handler.semaphores)

  7451. 

  7452.         self.executor_server.hold_jobs_in_build = False

  7453.         self.fake_nodepool.paused = False

  7454.         self.executor_server.release()

  7455.         self.waitUntilSettled()

  7456. 

  7457.     def test_semaphore_abandon_pending_execution(self):

  7458.         "Test abandon with job semaphores and pending job execution"

  7459. 

  7460.         # Pause the executor so it doesn't take any jobs.

  7461.         self.executor_server.pause()

  7462. 

  7463.         # Start merger as the paused executor won't take merge jobs.

  7464.         self._startMerger()

  7465. 

  7466.         # Pause nodepool so we can wait on the node requests and fulfill them

  7467.         # in a controlled manner.

  7468.         self.fake_nodepool.paused = True

  7469. 

  7470.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7471.         check_pipeline = tenant.layout.pipelines['check']

  7472. 

  7473.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  7474.         self.assertFalse('test-semaphore' in

  7475.                          tenant.semaphore_handler.semaphores)

  7476. 

  7477.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7478.         self.waitUntilSettled()

  7479.         self.assertEqual(len(self.scheds.first.sched.nodepool.requests), 2)

  7480. 

  7481.         # Now unpause nodepool to fulfill the node requests. We cannot use

  7482.         # waitUntilSettled here because the executor is paused.

  7483.         self.fake_nodepool.paused = False

  7484.         for _ in iterate_timeout(30, 'fulfill node requests'):

  7485.             if len(self.scheds.first.sched.nodepool.requests) == 0:

  7486.                 break

  7487. 

  7488.         self.assertTrue('test-semaphore' in

  7489.                         tenant.semaphore_handler.semaphores)

  7490. 

  7491.         self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())

  7492.         self.waitUntilSettled()

  7493. 

  7494.         # The check pipeline should be empty

  7495.         items = check_pipeline.getAllItems()

  7496.         self.assertEqual(len(items), 0)

  7497. 

  7498.         # The semaphore should be released

  7499.         self.assertFalse('test-semaphore' in

  7500.                          tenant.semaphore_handler.semaphores)

  7501. 

  7502.         self.executor_server.release()

  7503.         self.waitUntilSettled()

  7504. 

  7505.     def test_semaphore_new_patchset(self):

  7506.         "Test new patchset with job semaphores"

  7507.         self.executor_server.hold_jobs_in_build = True

  7508.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7509.         check_pipeline = tenant.layout.pipelines['check']

  7510. 

  7511.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  7512.         self.assertFalse('test-semaphore' in

  7513.                          tenant.semaphore_handler.semaphores)

  7514. 

  7515.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7516.         self.waitUntilSettled()

  7517. 

  7518.         self.assertTrue('test-semaphore' in

  7519.                         tenant.semaphore_handler.semaphores)

  7520.         semaphore = tenant.semaphore_handler.semaphores['test-semaphore']

  7521.         self.assertEqual(len(semaphore), 1)

  7522. 

  7523.         A.addPatchset()

  7524.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))

  7525.         self.waitUntilSettled()

  7526. 

  7527.         self.assertTrue('test-semaphore' in

  7528.                         tenant.semaphore_handler.semaphores)

  7529.         semaphore = tenant.semaphore_handler.semaphores['test-semaphore']

  7530.         self.assertEqual(len(semaphore), 1)

  7531. 

  7532.         items = check_pipeline.getAllItems()

  7533.         self.assertEqual(items[0].change.number, '1')

  7534.         self.assertEqual(items[0].change.patchset, '2')

  7535.         self.assertTrue(items[0].live)

  7536. 

  7537.         self.executor_server.hold_jobs_in_build = False

  7538.         self.executor_server.release()

  7539.         self.waitUntilSettled()

  7540. 

  7541.         # The semaphore should be released

  7542.         self.assertFalse('test-semaphore' in

  7543.                          tenant.semaphore_handler.semaphores)

  7544. 

  7545.     def test_semaphore_reconfigure(self):

  7546.         "Test reconfigure with job semaphores"

  7547.         self.executor_server.hold_jobs_in_build = True

  7548.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7549.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  7550.         self.assertFalse('test-semaphore' in

  7551.                          tenant.semaphore_handler.semaphores)

  7552. 

  7553.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7554.         self.waitUntilSettled()

  7555. 

  7556.         self.assertTrue('test-semaphore' in

  7557.                         tenant.semaphore_handler.semaphores)

  7558. 

  7559.         # reconfigure without layout change

  7560.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  7561.         self.waitUntilSettled()

  7562.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7563. 

  7564.         # semaphore still must be held

  7565.         self.assertTrue('test-semaphore' in

  7566.                         tenant.semaphore_handler.semaphores)

  7567. 

  7568.         self.commitConfigUpdate(

  7569.             'common-config',

  7570.             'config/semaphore/zuul-reconfiguration.yaml')

  7571.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  7572.         self.waitUntilSettled()

  7573.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7574. 

  7575.         self.executor_server.release('project-test1')

  7576.         self.waitUntilSettled()

  7577. 

  7578.         # There should be no builds anymore

  7579.         self.assertEqual(len(self.builds), 0)

  7580. 

  7581.         # The semaphore should be released

  7582.         self.assertFalse('test-semaphore' in

  7583.                          tenant.semaphore_handler.semaphores)

  7584. 

  7585.     def test_semaphore_reconfigure_job_removal(self):

  7586.         "Test job removal during reconfiguration with semaphores"

  7587.         self.executor_server.hold_jobs_in_build = True

  7588.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7589. 

  7590.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  7591.         self.assertFalse('test-semaphore' in

  7592.                          tenant.semaphore_handler.semaphores)

  7593. 

  7594.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7595.         self.waitUntilSettled()

  7596. 

  7597.         self.assertTrue('test-semaphore' in

  7598.                         tenant.semaphore_handler.semaphores)

  7599. 

  7600.         self.commitConfigUpdate(

  7601.             'common-config',

  7602.             'config/semaphore/git/common-config/zuul-remove-job.yaml')

  7603.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  7604.         self.waitUntilSettled()

  7605. 

  7606.         # Release job project-test1 which should be the only job left

  7607.         self.executor_server.release('project-test1')

  7608.         self.waitUntilSettled()

  7609. 

  7610.         # The check pipeline should be empty

  7611.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7612.         check_pipeline = tenant.layout.pipelines['check']

  7613.         items = check_pipeline.getAllItems()

  7614.         self.assertEqual(len(items), 0)

  7615. 

  7616.         # The semaphore should be released

  7617.         self.assertFalse('test-semaphore' in

  7618.                          tenant.semaphore_handler.semaphores)

  7619. 

  7620.         self.executor_server.hold_jobs_in_build = False

  7621.         self.executor_server.release()

  7622.         self.waitUntilSettled()

  7623. 

  7624.     def test_semaphore_reconfigure_job_removal_pending_node_request(self):

  7625.         """

  7626.         Test job removal during reconfiguration with semaphores and pending

  7627.         node request.

  7628.         """

  7629.         self.executor_server.hold_jobs_in_build = True

  7630. 

  7631.         # Pause nodepool so we can block the job in node request state during

  7632.         # reconfiguration.

  7633.         self.fake_nodepool.pause()

  7634. 

  7635.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7636. 

  7637.         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

  7638.         self.assertFalse('test-semaphore' in

  7639.                          tenant.semaphore_handler.semaphores)

  7640. 

  7641.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7642.         self.waitUntilSettled()

  7643. 

  7644.         self.assertTrue('test-semaphore' in

  7645.                         tenant.semaphore_handler.semaphores)

  7646. 

  7647.         self.commitConfigUpdate(

  7648.             'common-config',

  7649.             'config/semaphore/git/common-config/zuul-remove-job.yaml')

  7650.         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

  7651.         self.waitUntilSettled()

  7652. 

  7653.         # Now we can unpause nodepool

  7654.         self.fake_nodepool.unpause()

  7655.         self.waitUntilSettled()

  7656. 

  7657.         # Release job project-test1 which should be the only job left

  7658.         self.executor_server.release('project-test1')

  7659.         self.waitUntilSettled()

  7660. 

  7661.         # The check pipeline should be empty

  7662.         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7663.         check_pipeline = tenant.layout.pipelines['check']

  7664.         items = check_pipeline.getAllItems()

  7665.         self.assertEqual(len(items), 0)

  7666. 

  7667.         # The semaphore should be released

  7668.         self.assertFalse('test-semaphore' in

  7669.                          tenant.semaphore_handler.semaphores)

  7670. 

  7671.         self.executor_server.hold_jobs_in_build = False

  7672.         self.executor_server.release()

  7673.         self.waitUntilSettled()

  7674. 

  7675. 

  7676. class TestSemaphoreMultiTenant(ZuulTestCase):

  7677.     tenant_config_file = 'config/multi-tenant-semaphore/main.yaml'

  7678. 

  7679.     def test_semaphore_tenant_isolation(self):

  7680.         "Test semaphores in multiple tenants"

  7681. 

  7682.         self.waitUntilSettled()

  7683.         tenant_one = self.scheds.first.sched.abide.tenants.get('tenant-one')

  7684.         tenant_two = self.scheds.first.sched.abide.tenants.get('tenant-two')

  7685. 

  7686.         self.executor_server.hold_jobs_in_build = True

  7687.         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')

  7688.         B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')

  7689.         C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')

  7690.         D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')

  7691.         E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E')

  7692.         self.assertFalse('test-semaphore' in

  7693.                          tenant_one.semaphore_handler.semaphores)

  7694.         self.assertFalse('test-semaphore' in

  7695.                          tenant_two.semaphore_handler.semaphores)

  7696. 

  7697.         # add patches to project1 of tenant-one

  7698.         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))

  7699.         self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))

  7700.         self.waitUntilSettled()

  7701. 

  7702.         # one build of project1-test1 must run

  7703.         # semaphore of tenant-one must be acquired once

  7704.         # semaphore of tenant-two must not be acquired

  7705.         self