Check for stale PID lock when starting

The PID lock file does not actually test whether the lock is valid during
acquisition.  This leads to a failure to acquire the lock (essentially just
a file-based lock) when starting the process after a failure / kill.

Change-Id: Iebf0e077377278eb28b3280a8abfc605ac68a759

nodepool/cmd/nodepoold.py

@@ -16,6 +16,7 @@
 
 import argparse
 import daemon
+import errno
 import extras
 
 # as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
@@ -54,6 +55,28 @@ def stack_dump_handler(signum, frame):
     signal.signal(signal.SIGUSR2, stack_dump_handler)
 
 
+def is_pidfile_stale(pidfile):
+    """ Determine whether a PID file is stale.
+
+        Return 'True' ("stale") if the contents of the PID file are
+        valid but do not match the PID of a currently-running process;
+        otherwise return 'False'.
+
+        """
+    result = False
+
+    pidfile_pid = pidfile.read_pid()
+    if pidfile_pid is not None:
+        try:
+            os.kill(pidfile_pid, 0)
+        except OSError as exc:
+            if exc.errno == errno.ESRCH:
+                # The specified PID does not exist
+                result = True
+
+    return result
+
+
 class NodePoolDaemon(object):
     def __init__(self):
         self.args = None
@@ -118,6 +141,8 @@ def main():
         return(0)
 
     pid = pid_file_module.TimeoutPIDLockFile(npd.args.pidfile, 10)
+    if is_pidfile_stale(pid):
+        pid.break_lock()
 
     if npd.args.nodaemon:
         npd.main()