zuul
/
zuul
Code Issues Proposed changes

Always try to unlock nodes when returning 

Even if things are not in an expected state (eg, node not locked)
or we encounter errors updating node state, always try to unlock
the nodes when returning a nodeset.  Finally, if that too fails,
it will be logged but not treated as a fatal exception so that
we don't get stuck in queue processing.

Change-Id: Iab3bae4146c62067342e0c521a6289d045e65a5c
This commit is contained in:
James E. Blair 2017-09-29 07:46:31 -07:00
parent 66e0413803
commit d43ef51332
2 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
zuul/manager/__init__.py
View File

@ -407,13 +407,8 @@ class PipelineManager(object):
old_build_set.item, build.job)
if not was_running:
try:
nodeset = build.build_set.getJobNodeSet(build.job.name)
self.sched.nodepool.returnNodeSet(nodeset)
except Exception:
self.log.exception("Unable to return nodeset %s for "
"canceled build request %s" %
(nodeset, build))
nodeset = build.build_set.getJobNodeSet(build.job.name)
self.sched.nodepool.returnNodeSet(nodeset)
build.result = 'CANCELED'
canceled = True
canceled_jobs.add(build.job.name)

13
zuul/nodepool.py
View File

@ -90,10 +90,15 @@ class Nodepool(object):
self.log.info("Returning nodeset %s" % (nodeset,))
for node in nodeset.getNodes():
if node.lock is None:
raise Exception("Node %s is not locked" % (node,))
if node.state == model.STATE_IN_USE:
node.state = model.STATE_USED
self.sched.zk.storeNode(node)
self.log.error("Node %s is not locked" % (node,))
else:
try:
if node.state == model.STATE_IN_USE:
node.state = model.STATE_USED
self.sched.zk.storeNode(node)
except Exception:
self.log.exception("Exception storing node %s "
"while unlocking:" % (node,))
self._unlockNodes(nodeset.getNodes())
def unlockNodeSet(self, nodeset):