Accept SIGHUP for graceful shutdown

This patch introduces SIGHUP handler into agent processes, so that
operators can stop agent processes like storlet-daemon-factory
gracefully.
The SIGTERM signal is still available to stop agents immediately,
with interuppting all processes currently in progress.

Change-Id: Iafe9a9f846fc806398fd648c1ab39f95752236d7

storlets/agent/common/server.py

@@ -15,6 +15,7 @@
 from functools import partial
 import json
 import os
+import signal
 
 from storlets.sbus import SBus
 import storlets.sbus.command as sbus_cmd
@@ -80,6 +81,7 @@ class SBusServer(object):
     def __init__(self, sbus_path, logger):
         self.sbus_path = sbus_path
         self.logger = logger
+        self.running = True
         self.listen_timeout = LISTEN_TIMEOUT
         self.loop_timeout = LOOP_TIMEOUT
 
@@ -160,6 +162,23 @@ class SBusServer(object):
     def _terminate(self):
         raise NotImplementedError()
 
+    def _force_terminate(self):
+        raise NotImplementedError()
+
+    def _graceful_shutdown(self, *args):
+        self.logger.info("received SIGHUP. Shutting down gracefully")
+        self.running = False
+
+    def _force_shutdown(self, *args):
+        self.logger.info("received SIGTERM. Shutting down now")
+        try:
+            self._force_terminate()
+        except Exception:
+            self.logger.exception(
+                "Failed to force_terminate. The process stops anyway")
+        finally:
+            os._exit(1)
+
     def main_loop(self):
         """
         Main loop to run storlet application
@@ -173,10 +192,13 @@ class SBusServer(object):
             self.logger.error("Failed to create SBus. exiting.")
             return EXIT_FAILURE
 
+        signal.signal(signal.SIGHUP, self._graceful_shutdown)
+        signal.signal(signal.SIGTERM, self._force_shutdown)
+
         loop_cnt = 0
         status = EXIT_SUCCESS
 
-        while True:
+        while self.running:
             rc = sbus.listen(fd, self.loop_timeout)
 
             if rc < 0:
@@ -190,6 +212,7 @@ class SBusServer(object):
                     break
                 continue
 
+            # Reset loop_cnt here as a request comes via sbus
             loop_cnt = 0
 
             dtg = sbus.receive(fd)

storlets/agent/daemon/server.py

@@ -221,7 +221,7 @@ class StorletDaemon(SBusServer):
 
         pid = self.task_id_to_pid.get(task_id)
         try:
-            os.kill(pid, signal.SIGTERM)
+            os.kill(pid, signal.SIGKILL)
             self._remove_pid(pid)
             return CommandSuccess('Cancelled task %s' % task_id, False)
         except OSError:
@@ -236,6 +236,9 @@ class StorletDaemon(SBusServer):
     def _terminate(self):
         self._wait_all_child_processes()
 
+    def _force_terminate(self):
+        os.killpg(0, signal.SIGKILL)
+
 
 def main():
     """

storlets/agent/daemon_factory/server.py

@@ -515,7 +515,10 @@ class StorletDaemonFactory(SBusServer):
             return CommandFailure(err.args[0], False)
 
     def _terminate(self):
-        pass
+        self.shutdown_all_processes()
+
+    def _force_terminate(self):
+        self.process_kill_all()
 
 
 def main():

storlets/gateway/gateways/docker/runtime.py

@@ -323,7 +323,7 @@ class RunTimeSandbox(object):
             client.containers.run(
                 docker_image_name, detach=True, name=docker_container_name,
                 network_disabled=True, mounts=mounts, user='swift',
-                auto_remove=True)
+                auto_remove=True, stop_signal='SIGHUP')
         except docker.errors.ImageNotFound:
             msg = "Image %s is not found" % docker_image_name
             raise StorletRuntimeException(msg)