zuul
/
zuul
Code Issues Proposed changes

Cleanup stale locks in merger/executor API 

It's not clear why, but OpenDev currently has merger lock nodes
with no underlying requests.  It's possible these are leftover from
earlier versions of Zuul, or aborted processes, or something similarly
benign.  Regardless, under the principal that Zuul should clean up
after itself, let's automatically delete them.

Change-Id: I2dffcea3b1c154802bc1f3579d5e7888069d42fe
This commit is contained in:
James E. Blair 2021-08-19 15:08:37 -07:00
parent 6a0b5c419c
commit c08b09388b
2 changed files with 22 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
tests/unit/test_zk.py
View File

@ -984,6 +984,15 @@ class TestMergerApi(ZooKeeperBaseTestCase):
self.assertFalse(server.lock(server_a))
self._assertEmptyRoots(client)
def test_leaked_lock(self):
client = MergerApi(self.zk_client)
# Manually create a lock with no underlying request
self.zk_client.client.create(f"{client.LOCK_ROOT}/A", b'')
client.cleanup(0)
self._assertEmptyRoots(client)
def test_lost_merge_requests(self):
# Test that lostMergeRequests() returns unlocked running merge
# requests

13
zuul/zk/job_request_queue.py
View File

@ -486,6 +486,19 @@ class JobRequestQueue(ZooKeeperSimpleBase):
except Exception:
self.log.exception(
"Error cleaning up result queue %s", self)
try:
for lock_id in self.kazoo_client.get_children(self.LOCK_ROOT):
try:
lock_path = "/".join([self.LOCK_ROOT, lock_id])
request_path = "/".join([self.REQUEST_ROOT, lock_id])
if not self.kazoo_client.exists(request_path):
self.log.error("Removing stale lock: %s", lock_path)
self.kazoo_client.delete(lock_path, recursive=True)
except Exception:
self.log.execption(
"Unable to delete lock %s", path)
except Exception:
self.log.exception("Error cleaning up locks %s", self)
@staticmethod
def _bytesToDict(data):