From e3c952ef3a1aaf55afbe302cdb16abcc7b8d58cb Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Thu, 25 Jan 2018 10:08:03 -0500 Subject: [PATCH] Handle missing request during a decline. We've seen a handler continually decline a request because the request went missing and we didn't handle that exception, so we never set the handler to 'done'. This handles that exception, and adds a safety mechanism to decline_request(). Change-Id: Ieb06846e0fe2aadfd9df0031e58b2df98bbfcc03 --- nodepool/driver/__init__.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/nodepool/driver/__init__.py b/nodepool/driver/__init__.py index b44d67896..b90f2f099 100644 --- a/nodepool/driver/__init__.py +++ b/nodepool/driver/__init__.py @@ -212,7 +212,11 @@ class NodeRequestHandler(object): self.nodeset = [] def decline_request(self): - self.request.declined_by.append(self.launcher_id) + # Technically, this check to see if we've already declined it should + # not be necessary. But if there is a bug (and there has been), we + # want to make sure we don't continuously grow this array. + if self.launcher_id not in self.request.declined_by: + self.request.declined_by.append(self.launcher_id) launchers = set(self.zk.getRegisteredLaunchers()) if launchers.issubset(set(self.request.declined_by)): # All launchers have declined it @@ -239,8 +243,14 @@ class NodeRequestHandler(object): "NodeRequestHandler:", self.request.id) self.decline_request() self.unlockNodeSet(clear_allocation=True) - self.zk.storeNodeRequest(self.request) - self.zk.unlockNodeRequest(self.request) + try: + self.zk.storeNodeRequest(self.request) + self.zk.unlockNodeRequest(self.request) + except Exception: + # If the request is gone for some reason, we need to make + # sure that self.done still gets set. + self.log.exception("Unable to decline missing request %s", + self.request.id) self.done = True def poll(self):