zuul
/
zuul
Code Issues Proposed changes

Log zkobject paths with errors 

We encountered an issue in OpenDev where Zuul was unable to load
a queue item from zk because the object was missing (NoNodeError)
but we did not log which object it was, so further debugging was
impractical.  To address this, log the znode path.

Change-Id: I25a4702adb65892c1d70db6bc560eb98a5cc205a
This commit is contained in:
James E. Blair 2021-12-02 09:09:52 -08:00
parent 58252504c7
commit ba5db9c431
1 changed files with 20 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
zuul/zk/zkobject.py
View File

@ -173,6 +173,8 @@ class ZKObject:
# These errors come from the server and are not
# retryable. Connection errors are KazooExceptions so
# they aren't caught here and we will retry.
context.log.error(
"Exception deleting ZKObject %s at %s", self, path)
raise
except KazooException:
context.log.exception(
@ -201,16 +203,19 @@ class ZKObject:
# These errors come from the server and are not
# retryable. Connection errors are KazooExceptions so
# they aren't caught here and we will retry.
context.log.error(
"Exception loading ZKObject %s at %s", self, path)
raise
except KazooException:
context.log.exception(
"Exception loading ZKObject %s, will retry", self)
"Exception loading ZKObject %s at %s, will retry",
self, path)
time.sleep(5)
except Exception:
# A higher level must handle this exception, but log
# ourself here so we know what object triggered it.
context.log.error(
"Exception loading ZKObject %s", self)
"Exception loading ZKObject %s at %s", self, path)
raise
raise Exception("ZooKeeper session or lock not valid")
@ -232,10 +237,13 @@ class ZKObject:
# These errors come from the server and are not
# retryable. Connection errors are KazooExceptions so
# they aren't caught here and we will retry.
context.log.error(
"Exception saving ZKObject %s at %s", self, path)
raise
except KazooException:
context.log.exception(
"Exception saving ZKObject %s, will retry", self)
"Exception saving ZKObject %s at %s, will retry",
self, path)
time.sleep(self._retry_interval)
raise Exception("ZooKeeper session or lock not valid")
@ -276,16 +284,19 @@ class ShardedZKObject(ZKObject):
# These errors come from the server and are not
# retryable. Connection errors are KazooExceptions so
# they aren't caught here and we will retry.
context.log.error(
"Exception loading ZKObject %s at %s", self, path)
raise
except KazooException:
context.log.exception(
"Exception loading ZKObject %s, will retry", self)
"Exception loading ZKObject %s at %s, will retry",
self, path)
time.sleep(5)
except Exception as exc:
# A higher level must handle this exception, but log
# ourself here so we know what object triggered it.
context.log.error(
"Exception loading ZKObject %s", self)
"Exception loading ZKObject %s at %s", self, path)
if self.delete_on_error:
self.delete(context)
raise InvalidObjectError from exc
@ -310,9 +321,12 @@ class ShardedZKObject(ZKObject):
# These errors come from the server and are not
# retryable. Connection errors are KazooExceptions so
# they aren't caught here and we will retry.
context.log.error(
"Exception saving ZKObject %s at %s", self, path)
raise
except KazooException:
context.log.exception(
"Exception saving ZKObject %s, will retry", self)
"Exception saving ZKObject %s at %s, will retry",
self, path)
time.sleep(self._retry_interval)
raise Exception("ZooKeeper session or lock not valid")