diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 4a4d6de00c..5a265bea93 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -1112,6 +1112,7 @@ class TestScheduler(testtools.TestCase):
                     self.sched.result_event_queue.empty() and
                     self.fake_gerrit.event_queue.empty() and
                     not self.merge_client.build_sets and
+                    self.haveAllBuildsReported() and
                     self.areAllBuildsWaiting()):
                     self.sched.run_handler_lock.release()
                     self.worker.lock.release()
diff --git a/zuul/launcher/gearman.py b/zuul/launcher/gearman.py
index 0e248af493..3638add3e8 100644
--- a/zuul/launcher/gearman.py
+++ b/zuul/launcher/gearman.py
@@ -352,11 +352,11 @@ class Gearman(object):
         self.log.debug("Still unable to find build %s to cancel" % build)
         if build.number:
             self.log.debug("Build %s has just started" % build)
-            self.cancelRunningBuild(build)
-            self.log.debug("Canceled just running build %s" % build)
         else:
-            self.log.error("Build %s has not started but "
-                           "was not found in queue" % build)
+            self.log.error("Build %s has not started but was not"
+                           "found in queue; canceling anyway" % build)
+        self.cancelRunningBuild(build)
+        self.log.debug("Canceled possibly running build %s" % build)
 
     def onBuildCompleted(self, job, result=None):
         if job.unique in self.meta_jobs: