Fix Graceful shutdown timeout with parent died

cotyledon/__init__.py

@@ -144,16 +144,19 @@ class Service(object):
         with _exit_on_exception():
             self.reload()
 
-    def _clean_exit(self, *args, **kwargs):
+    def _terminate(self):
         signal.signal(signal.SIGTERM, signal.SIG_IGN)
         if self.graceful_shutdown_timeout > 0:
             signal.alarm(self.graceful_shutdown_timeout)
-        LOG.info('Caught SIGTERM signal, '
-                 'graceful exiting of service %s' % self._title)
         with _exit_on_exception():
             self.terminate()
             sys.exit(0)
 
+    def _clean_exit(self, *args, **kwargs):
+        LOG.info('Caught SIGTERM signal, '
+                 'graceful exiting of service %s' % self._title)
+        self._terminate()
+
     def _graceful_shutdown_timeout_cb(self, signum, frame):
         LOG.info('Graceful shutdown timeout (%d) exceeded, exiting %s now.' %
                  (self.graceful_shutdown_timeout, self._title))
@@ -464,10 +467,7 @@ class ServiceManager(object):
         if self._current_process is not None:
             LOG.info('Parent process has died unexpectedly, %s exiting'
                      % self._current_process._title)
-            with _exit_on_exception():
+            os.kill(os.getpid(), signal.SIGTERM)
-                self._current_process.terminate()
-                sys.exit(0)
-
         else:
             os._exit(0)
 

cotyledon/tests/test_cotyledon.py

@@ -190,6 +190,12 @@ class TestCotyledon(Base):
         self.assertEqual([
             b'ERROR:cotyledon.tests.examples:heavy terminate',
             b'ERROR:cotyledon.tests.examples:heavy terminate',
+            b'INFO:cotyledon:Caught SIGTERM signal, graceful exiting of '
+            b'service heavy(0) [XXXX]',
+            b'INFO:cotyledon:Caught SIGTERM signal, graceful exiting of '
+            b'service heavy(1) [XXXX]',
+            b'INFO:cotyledon:Caught SIGTERM signal, graceful exiting of '
+            b'service light(0) [XXXX]',
             b'INFO:cotyledon:Parent process has died unexpectedly, '
             b'heavy(0) [XXXX] exiting',
             b'INFO:cotyledon:Parent process has died unexpectedly, '
@@ -206,7 +212,7 @@ class TestBuggyCotyledon(Base):
     @unittest.skipIf(sys.version_info[0] != 3,
                      "Buggy on py27, time.sleep returns before alarm callback "
                      "is called")
-    def test_graceful_timeout(self):
+    def test_graceful_timeout_term(self):
         lines = self.get_lines(1)
         childpid = self.get_pid(lines[0])
         self.subp.terminate()
@@ -218,10 +224,35 @@ class TestBuggyCotyledon(Base):
         self.assertNotIn('ERROR:cotyledon.tests.examples:time.sleep done',
                          lines)
         self.assertEqual([
+            b'INFO:cotyledon:Caught SIGTERM signal, graceful exiting of '
+            b'service buggy(0) [XXXX]',
             b'INFO:cotyledon:Graceful shutdown timeout (1) exceeded, '
             b'exiting buggy(0) [XXXX] now.',
             b'DEBUG:cotyledon:Shutdown finish'
-        ], lines[-2:])
+        ], lines[-3:])
+
+    @unittest.skipIf(sys.version_info[0] != 3,
+                     "Buggy on py27, time.sleep returns before alarm callback "
+                     "is called")
+    def test_graceful_timeout_kill(self):
+        lines = self.get_lines(1)
+        childpid = self.get_pid(lines[0])
+        self.subp.kill()
+        time.sleep(2)
+        self.assertEqual(-9, self.subp.poll())
+        self.assertRaises(OSError, os.kill, self.subp.pid, 0)
+        self.assertRaises(OSError, os.kill, childpid, 0)
+        lines = self.hide_pids(self.get_lines())
+        self.assertNotIn('ERROR:cotyledon.tests.examples:time.sleep done',
+                         lines)
+        self.assertEqual([
+            b'INFO:cotyledon:Parent process has died unexpectedly, buggy(0) '
+            b'[XXXX] exiting',
+            b'INFO:cotyledon:Caught SIGTERM signal, graceful exiting of '
+            b'service buggy(0) [XXXX]',
+            b'INFO:cotyledon:Graceful shutdown timeout (1) exceeded, '
+            b'exiting buggy(0) [XXXX] now.',
+        ], lines[-3:])
 
 
 class TestOsloCotyledon(Base):