diff --git a/tests/fixtures/layout-live-reconfiguration-del-project.yaml b/tests/fixtures/layout-live-reconfiguration-del-project.yaml new file mode 100644 index 0000000000..07ffb2e136 --- /dev/null +++ b/tests/fixtures/layout-live-reconfiguration-del-project.yaml @@ -0,0 +1,21 @@ +pipelines: + - name: check + manager: IndependentPipelineManager + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + verified: 1 + failure: + gerrit: + verified: -1 + +projects: + - name: org/project + merge-mode: cherry-pick + check: + - project-merge: + - project-test1 + - project-test2 + - project-testfile diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 61a2d09173..7ed7efb280 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2494,6 +2494,57 @@ class TestScheduler(ZuulTestCase): # Ensure the removed job was not included in the report. self.assertNotIn('project1-project2-integration', A.messages[0]) + def test_live_reconfiguration_del_project(self): + # Test project deletion from layout + # while changes are enqueued + + self.worker.hold_jobs_in_build = True + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') + C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C') + + # A Depends-On: B + A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % ( + A.subject, B.data['id']) + self.fake_gerrit.addEvent(B.addApproval('APRV', 1)) + + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.worker.release('.*-merge') + self.waitUntilSettled() + self.assertEqual(len(self.builds), 5) + + # This layout defines only org/project, not org/project1 + self.config.set('zuul', 'layout_config', + 'tests/fixtures/layout-live-' + 'reconfiguration-del-project.yaml') + self.sched.reconfigure(self.config) + self.waitUntilSettled() + + # Builds for C aborted, builds for A succeed, + # and have change B applied ahead + job_c = self.getJobFromHistory('project1-test1') + self.assertEqual(job_c.changes, '3,1') + self.assertEqual(job_c.result, 'ABORTED') + + self.worker.hold_jobs_in_build = False + self.worker.release() + self.waitUntilSettled() + + self.assertEqual(self.getJobFromHistory('project-test1').changes, + '2,1 1,1') + + self.assertEqual(A.data['status'], 'NEW') + self.assertEqual(B.data['status'], 'NEW') + self.assertEqual(C.data['status'], 'NEW') + self.assertEqual(A.reported, 1) + self.assertEqual(B.reported, 0) + self.assertEqual(C.reported, 0) + + self.assertEqual(len(self.sched.layout.pipelines['check'].queues), 0) + self.assertIn('Build succeeded', A.messages[0]) + def test_live_reconfiguration_functions(self): "Test live reconfiguration with a custom function" self.worker.registerFunction('build:node-project-test1:debian') @@ -3668,6 +3719,48 @@ For CI problems and help debugging, contact ci@example.org""" self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'NEW') + def test_crd_gate_unknown(self): + "Test unknown projects in dependent pipeline" + self.init_repo("org/unknown") + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'B') + A.addApproval('CRVW', 2) + B.addApproval('CRVW', 2) + + # A Depends-On: B + A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % ( + A.subject, B.data['id']) + + B.addApproval('APRV', 1) + self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) + self.waitUntilSettled() + + # Unknown projects cannot share a queue with any other + # since they don't have common jobs with any other (they have no jobs). + # Changes which depend on unknown project changes + # should not be processed in dependent pipeline + self.assertEqual(A.data['status'], 'NEW') + self.assertEqual(B.data['status'], 'NEW') + self.assertEqual(A.reported, 0) + self.assertEqual(B.reported, 0) + self.assertEqual(len(self.history), 0) + + # Simulate change B being gated outside this layout + self.fake_gerrit.addEvent(B.addApproval('APRV', 1)) + B.setMerged() + self.waitUntilSettled() + self.assertEqual(len(self.history), 0) + + # Now that B is merged, A should be able to be enqueued and + # merged. + self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) + self.waitUntilSettled() + + self.assertEqual(A.data['status'], 'MERGED') + self.assertEqual(A.reported, 2) + self.assertEqual(B.data['status'], 'MERGED') + self.assertEqual(B.reported, 0) + def test_crd_check(self): "Test cross-repo dependencies in independent pipelines" @@ -3782,12 +3875,12 @@ For CI problems and help debugging, contact ci@example.org""" self.assertIn('Build succeeded', A.messages[0]) self.assertIn('Build succeeded', B.messages[0]) - def test_crd_check_reconfiguration(self): + def _test_crd_check_reconfiguration(self, project1, project2): "Test cross-repo dependencies re-enqueued in independent pipelines" self.gearman_server.hold_jobs_in_queue = True - A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') - B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + A = self.fake_gerrit.addFakeChange(project1, 'master', 'A') + B = self.fake_gerrit.addFakeChange(project2, 'master', 'B') # A Depends-On: B A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % ( @@ -3820,6 +3913,17 @@ For CI problems and help debugging, contact ci@example.org""" self.assertEqual(self.history[0].changes, '2,1 1,1') self.assertEqual(len(self.sched.layout.pipelines['check'].queues), 0) + def test_crd_check_reconfiguration(self): + self._test_crd_check_reconfiguration('org/project1', 'org/project2') + + def test_crd_undefined_project(self): + """Test that undefined projects in dependencies are handled for + independent pipelines""" + # It's a hack for fake gerrit, + # as it implies repo creation upon the creation of any change + self.init_repo("org/unknown") + self._test_crd_check_reconfiguration('org/project1', 'org/unknown') + def test_crd_check_ignore_dependencies(self): "Test cross-repo dependencies can be ignored" self.config.set('zuul', 'layout_config', diff --git a/zuul/model.py b/zuul/model.py index f8e0d25c0e..3a228f56ab 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -431,9 +431,13 @@ class ChangeQueue(object): class Project(object): - def __init__(self, name): + def __init__(self, name, foreign=False): self.name = name self.merge_mode = MERGER_MERGE_RESOLVE + # foreign projects are those referenced in dependencies + # of layout projects, this should matter + # when deciding whether to enqueue their changes + self.foreign = foreign def __str__(self): return self.name diff --git a/zuul/scheduler.py b/zuul/scheduler.py index a9bd6b21f9..4375f98c04 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -507,11 +507,15 @@ class Scheduler(threading.Thread): name = reporter.name self.reporters[name] = reporter - def getProject(self, name): + def getProject(self, name, create_foreign=False): self.layout_lock.acquire() p = None try: p = self.layout.projects.get(name) + if p is None and create_foreign: + self.log.info("Registering foreign project: %s" % name) + p = Project(name, foreign=True) + self.layout.projects[name] = p finally: self.layout_lock.release() return p @@ -685,15 +689,15 @@ class Scheduler(threading.Thread): item.items_behind = [] item.pipeline = None item.queue = None - project = layout.projects.get(item.change.project.name) - if not project: - self.log.warning("Unable to find project for " - "change %s while reenqueueing" % - item.change) - item.change.project = None - items_to_remove.append(item) - continue - item.change.project = project + project_name = item.change.project.name + item.change.project = layout.projects.get(project_name) + if not item.change.project: + self.log.debug("Project %s not defined, " + "re-instantiating as foreign" % + project_name) + project = Project(project_name, foreign=True) + layout.projects[project_name] = project + item.change.project = project item_jobs = new_pipeline.getJobs(item) for build in item.current_build_set.getBuilds(): job = layout.jobs.get(build.job.name) @@ -861,7 +865,7 @@ class Scheduler(threading.Thread): self.log.debug("Processing trigger event %s" % event) try: project = self.layout.projects.get(event.project_name) - if not project: + if not project or project.foreign: self.log.debug("Project %s not found" % event.project_name) return @@ -1797,10 +1801,11 @@ class IndependentPipelineManager(BasePipelineManager): if existing: return DynamicChangeQueueContextManager(existing) if change.project not in self.pipeline.getProjects(): - return DynamicChangeQueueContextManager(None) + self.pipeline.addProject(change.project) change_queue = ChangeQueue(self.pipeline) change_queue.addProject(change.project) self.pipeline.addQueue(change_queue) + self.log.debug("Dynamically created queue %s", change_queue) return DynamicChangeQueueContextManager(change_queue) def enqueueChangesAhead(self, change, quiet, ignore_requirements, diff --git a/zuul/trigger/gerrit.py b/zuul/trigger/gerrit.py index 175e3f8c62..05d75819e4 100644 --- a/zuul/trigger/gerrit.py +++ b/zuul/trigger/gerrit.py @@ -94,7 +94,7 @@ class GerritEventConnector(threading.Thread): Can not get account information." % event.type) event.account = None - if event.change_number: + if event.change_number and self.sched.getProject(event.project_name): # Call _getChange for the side effect of updating the # cache. Note that this modifies Change objects outside # the main thread. @@ -404,7 +404,11 @@ class Gerrit(object): if 'project' not in data: raise Exception("Change %s,%s not found" % (change.number, change.patchset)) - change.project = self.sched.getProject(data['project']) + # If updated changed came as a dependent on + # and its project is not defined, + # then create a 'foreign' project for it in layout + change.project = self.sched.getProject(data['project'], + create_foreign=bool(history)) change.branch = data['branch'] change.url = data['url'] max_ps = 0