Merge "Remove the local builds list from the executor client"

tests/base.py

@@ -5314,6 +5314,13 @@ class ZuulTestCase(BaseTestCase):
 
         return sorted(self.builds, key=lambda x: x.name)
 
+    def getCurrentBuilds(self):
+        for tenant in self.scheds.first.sched.abide.tenants.values():
+            for pipeline in tenant.layout.pipelines.values():
+                for item in pipeline.getAllItems():
+                    for build in item.current_build_set.builds.values():
+                        yield build
+
     def release(self, job):
         job.release()
 

tests/unit/test_inventory.py

@@ -72,11 +72,9 @@ class TestInventoryBase(ZuulTestCase):
         self.waitUntilSettled()
 
     def cancelExecutorJobs(self):
-        for app in self.scheds:
-            executor_client = app.sched.executor
-            builds = [b for b in executor_client.builds.values()]
-            for build in builds:
-                executor_client.cancel(build)
+        executor_client = self.scheds.first.sched.executor
+        for build in self.getCurrentBuilds():
+            executor_client.cancel(build)
 
 
 class TestInventoryGithub(TestInventoryBase):

tests/unit/test_scheduler.py

@@ -2023,7 +2023,7 @@ class TestScheduler(ZuulTestCase):
         self.waitUntilSettled()
 
         # Get the build request object
-        build = list(self.scheds.first.sched.executor.builds.values())[0]
+        build = list(self.getCurrentBuilds())[0]
 
         # We should report using the held node's resources
         self.waitUntilNodeCacheSync(
@@ -2937,7 +2937,7 @@ class TestScheduler(ZuulTestCase):
 
             # Wait until the BuildStartedEvent was processed by the scheduler
             # as we need the worker info in order to cancel a build.
-            builds = self.scheds.first.sched.executor.builds.values()
+            builds = self.getCurrentBuilds()
             if all(b.worker.name != "Unknown" for b in builds):
                 break
 
@@ -5615,17 +5615,18 @@ class TestScheduler(ZuulTestCase):
         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
         self.waitUntilSettled()
 
-        self.assertEqual(len(self.scheds.first.sched.executor.builds), 1)
+        current_builds = list(self.getCurrentBuilds())
+        self.assertEqual(len(current_builds), 1)
 
         self.log.debug('Current builds:')
-        self.log.debug(self.scheds.first.sched.executor.builds)
+        self.log.debug(current_builds)
 
         start = time.time()
         while True:
             if time.time() - start > 10:
                 raise Exception("Timeout waiting for gearman server to report "
                                 + "back to the client")
-            build = list(self.scheds.first.sched.executor.builds.values())[0]
+            build = list(self.getCurrentBuilds())[0]
             if build.worker.name == self.executor_server.hostname:
                 break
             else:
@@ -5804,13 +5805,18 @@ For CI problems and help debugging, contact ci@example.org"""
                                           self.gearman_server.port)
         self.addCleanup(client.shutdown)
 
-        # Wait for gearman server to send the initial workData back to zuul
+        # Wait for executor server to send the initial workData back to zuul
         start = time.time()
         while True:
             if time.time() - start > 10:
                 raise Exception("Timeout waiting for gearman server to report "
                                 + "back to the client")
-            build = list(self.scheds.first.sched.executor.builds.values())[0]
+            build = list(self.getCurrentBuilds())[0]
+            # TODO (felix): If we don't want to look up the builds from the
+            # pipelines/queues, we could use a DataWatch to listen for
+            # BuildStatus events (which set the worker info). However, this
+            # solution might not be as straight forward and more error prone
+            # due to the nature of watches.
             if build.worker.name == self.executor_server.hostname:
                 break
             else:

zuul/executor/client.py

@@ -14,7 +14,6 @@
 
 import logging
 import time
-from contextlib import suppress
 from uuid import uuid4
 
 import zuul.executor.common
@@ -39,8 +38,6 @@ class ExecutorClient(object):
     def __init__(self, config, sched):
         self.config = config
         self.sched = sched
-        self.builds = {}
-        self.meta_jobs = {}  # A list of meta-jobs like stop or describe
 
         self.executor_api = self._executor_api_class(self.sched.zk_client)
         self.result_events = PipelineResultEventQueue.createRegistry(
@@ -77,8 +74,6 @@ class ExecutorClient(object):
         log.debug("Adding build %s of job %s to item %s",
                   build, job, item)
         item.addBuild(build)
-        # TODO (felix): Remove once we are not relying on this list anymore
-        self.builds[uuid] = build
 
         if job.name == 'noop':
             data = {"start_time": time.time()}
@@ -197,7 +192,7 @@ class ExecutorClient(object):
 
         return False
 
-    def resumeBuild(self, build: Build) -> bool:
+    def resumeBuild(self, build):
         log = get_annotated_logger(self.log, build.zuul_event_id)
 
         if not build.build_request_ref:
@@ -211,13 +206,10 @@ class ExecutorClient(object):
             return True
         return False
 
-    def removeBuild(self, build: Build) -> None:
+    def removeBuild(self, build):
         log = get_annotated_logger(self.log, build.zuul_event_id)
         log.debug("Removing build %s", build.uuid)
 
-        with suppress(KeyError):
-            del self.builds[build.uuid]
-
         if not build.build_request_ref:
             log.debug("Build %s has not been submitted to ZooKeeper",
                       build.uuid)