Unlock request if it disappears

Found an issue where we were not unlocking the node request if it
disappeared on us. This caused the request lock cleanup to fail b/c
it remained lock.

Also, let's catch cleanup errors individually so that each phase has
a chance to run, independent of errors from other phases.

Also add recursive=True to the request lock delete.

Change-Id: I12c79b7725460eae5a27063523f3fa2e19e6bc59

nodepool/nodepool.py

@@ -842,6 +842,7 @@ class NodeRequestHandler(object):
                 node.allocated_to = None
                 self.zk.storeNode(node)
             self.unlockNodeSet()
+            self.zk.unlockNodeRequest(self.request)
             return True
 
         if self.launch_manager.failed_nodes:
@@ -1205,12 +1206,27 @@ class CleanupWorker(BaseCleanupWorker):
                 manager.cleanupLeakedFloaters()
 
     def _run(self):
+        '''
+        Catch exceptions individually so that other cleanup routines may
+        have a chance.
+        '''
         try:
             self._cleanupNodeRequestLocks()
+        except Exception:
+            self.log.exception(
+                "Exception in DeletedNodeWorker (node request lock cleanup):")
+
+        try:
             self._cleanupLeakedInstances()
+        except Exception:
+            self.log.exception(
+                "Exception in DeletedNodeWorker (leaked instance cleanup):")
+
+        try:
             self._cleanupLostRequests()
         except Exception:
-            self.log.exception("Exception in DeletedNodeWorker:")
+            self.log.exception(
+                "Exception in DeletedNodeWorker (lost request cleanup):")
 
 
 class DeletedNodeWorker(BaseCleanupWorker):

nodepool/zk.py

@@ -1308,7 +1308,7 @@ class ZooKeeper(object):
         '''
         path = self._requestLockPath(lock)
         try:
-            self.client.delete(path)
+            self.client.delete(path, recursive=True)
         except kze.NoNodeError:
             pass