Cancel obsolete builds on reconfiguration

Cleanup running builds for jobs that are no longer defined on
reconfiguration.

Change-Id: I746a5ba8d9034ae846fd77080af2bfabc8aedf44

tests/fixtures/layout-no-jobs.yaml

@@ -0,0 +1,43 @@
+includes:
+  - python-file: custom_functions.py
+
+pipelines:
+  - name: check
+    manager: IndependentPipelineManager
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+  - name: gate
+    manager: DependentPipelineManager
+    failure-message: Build failed.  For information on how to proceed, see http://wiki.example.org/Test_Failures
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - approved: 1
+    success:
+      gerrit:
+        verified: 2
+        submit: true
+    failure:
+      gerrit:
+        verified: -2
+    start:
+      gerrit:
+        verified: 0
+    precedence: high
+
+projects:
+  - name: org/project
+    merge-mode: cherry-pick
+    check:
+      - noop
+    gate:
+      - noop

tests/fixtures/layout.yaml

@@ -231,3 +231,7 @@ projects:
       - conflict-project-merge:
         - conflict-project-test1
         - conflict-project-test2
+
+  - name: org/noop-project
+    gate:
+      - noop

tests/test_scheduler.py

@@ -2813,6 +2813,29 @@ class TestScheduler(testtools.TestCase):
         self.assertReportedStat('test-timing', '3|ms')
         self.assertReportedStat('test-guage', '12|g')
 
+    def test_stuck_job_cleanup(self):
+        "Test that pending jobs are cleaned up if removed from layout"
+        self.gearman_server.hold_jobs_in_queue = True
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        A.addApproval('CRVW', 2)
+        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+        self.waitUntilSettled()
+        self.assertEqual(len(self.gearman_server.getQueue()), 1)
+
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-no-jobs.yaml')
+        self.sched.reconfigure(self.config)
+        self.waitUntilSettled()
+
+        self.gearman_server.release('noop')
+        self.waitUntilSettled()
+        self.assertEqual(len(self.gearman_server.getQueue()), 0)
+        self.assertTrue(self.sched._areAllBuildsComplete())
+
+        self.assertEqual(len(self.history), 1)
+        self.assertEqual(self.history[0].name, 'noop')
+        self.assertEqual(self.history[0].result, 'SUCCESS')
+
     def test_file_jobs(self):
         "Test that file jobs run only when appropriate"
         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')

zuul/scheduler.py

@@ -565,8 +565,7 @@ class Scheduler(threading.Thread):
                         self.log.warning("No old pipeline matching %s found "
                                          "when reconfiguring" % name)
                     continue
-                self.log.debug("Re-enqueueing changes for pipeline %s" %
-                               name)
+                self.log.debug("Re-enqueueing changes for pipeline %s" % name)
                 items_to_remove = []
                 for shared_queue in old_pipeline.queues:
                     for item in shared_queue.queue:
@@ -582,16 +581,30 @@ class Scheduler(threading.Thread):
                             items_to_remove.append(item)
                             continue
                         item.change.project = project
+                        for build in item.current_build_set.getBuilds():
+                            build.job = layout.jobs.get(build.job.name,
+                                                        build.job)
                         if not new_pipeline.manager.reEnqueueItem(item):
                             items_to_remove.append(item)
                 builds_to_remove = []
                 for build, item in old_pipeline.manager.building_jobs.items():
                     if item in items_to_remove:
                         builds_to_remove.append(build)
-                        self.log.warning("Deleting running build %s for "
-                                         "change %s while reenqueueing" % (
-                                         build, item.change))
+                        self.log.warning(
+                            "Deleting running build %s for change %s whose "
+                            "item was not re-enqueued" % (build, item.change))
+                    if build.job not in new_pipeline.getJobs(item.change):
+                        builds_to_remove.append(build)
+                        self.log.warning(
+                            "Deleting running build %s for change %s because "
+                            "the job is not defined" % (build, item.change))
                 for build in builds_to_remove:
+                    try:
+                        self.launcher.cancel(build)
+                    except Exception:
+                        self.log.exception(
+                            "Exception while canceling build %s "
+                            "for change %s" % (build, item.change))
                     del old_pipeline.manager.building_jobs[build]
                 new_pipeline.manager.building_jobs = \
                     old_pipeline.manager.building_jobs