diff --git a/swift/common/daemon.py b/swift/common/daemon.py index 300710e982..d6c431b6d1 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -159,7 +159,7 @@ class DaemonStrategy(object): except KeyboardInterrupt: self.logger.notice('User quit') finally: - self.cleanup() + self.cleanup(stopping=True) self.running = False def _fork(self, once, **kwargs): @@ -167,6 +167,8 @@ class DaemonStrategy(object): if pid == 0: signal.signal(signal.SIGHUP, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) + # only MAINPID should be sending notifications + os.environ.pop('NOTIFY_SOCKET', None) self.daemon.run(once, **kwargs) @@ -245,7 +247,15 @@ class DaemonStrategy(object): self.daemon.post_multiprocess_run() return 0 - def cleanup(self): + def cleanup(self, stopping=False): + """ + Cleanup worker processes + + :param stopping: if set, tell systemd we're stopping + """ + + if stopping: + utils.systemd_notify(self.logger, "STOPPING=1") for p in self.spawned_pids(): try: os.kill(p, signal.SIGTERM) diff --git a/swift/common/utils/__init__.py b/swift/common/utils/__init__.py index 6af137bb2e..8f15756c19 100644 --- a/swift/common/utils/__init__.py +++ b/swift/common/utils/__init__.py @@ -6206,17 +6206,28 @@ def get_db_files(db_path): return sorted(results) -def systemd_notify(logger=None): +def systemd_notify(logger=None, msg=b"READY=1"): """ - Notify the service manager that started this process, if it is - systemd-compatible, that this process correctly started. To do so, - it communicates through a Unix socket stored in environment variable - NOTIFY_SOCKET. More information can be found in systemd documentation: + Send systemd-compatible notifications. + + Notify the service manager that started this process, if it has set the + NOTIFY_SOCKET environment variable. For example, systemd will set this + when the unit has ``Type=notify``. More information can be found in + systemd documentation: https://www.freedesktop.org/software/systemd/man/sd_notify.html + Common messages include:: + + READY=1 + RELOADING=1 + STOPPING=1 + STATUS= + :param logger: a logger object + :param msg: the message to send """ - msg = b'READY=1' + if not isinstance(msg, bytes): + msg = msg.encode('utf8') notify_socket = os.getenv('NOTIFY_SOCKET') if notify_socket: if notify_socket.startswith('@'): @@ -6227,7 +6238,6 @@ def systemd_notify(logger=None): try: sock.connect(notify_socket) sock.sendall(msg) - del os.environ['NOTIFY_SOCKET'] except EnvironmentError: if logger: logger.debug("Systemd notification failed", exc_info=True) diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 93a4dad3d7..c9a100b637 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -487,6 +487,8 @@ class StrategyBase(object): capture_stdio(self.logger) drop_privileges(self.conf.get('user', 'swift')) del self.tracking_data # children don't need to track siblings + # only MAINPID should be sending systemd notifications + os.environ.pop('NOTIFY_SOCKET', None) def shutdown_sockets(self): """ @@ -888,6 +890,7 @@ def run_wsgi(conf_path, app_section, *args, **kwargs): run_server(conf, logger, no_fork_sock, global_conf=global_conf, ready_callback=strategy.signal_ready, allow_modify_pipeline=allow_modify_pipeline) + systemd_notify(logger, "STOPPING=1") return 0 def stop_with_signal(signum, *args): @@ -981,8 +984,10 @@ def run_wsgi(conf_path, app_section, *args, **kwargs): else: logger.notice('%s received (%s)', signame, os.getpid()) if running_context[1] == signal.SIGTERM: + systemd_notify(logger, "STOPPING=1") os.killpg(0, signal.SIGTERM) elif running_context[1] == signal.SIGUSR1: + systemd_notify(logger, "RELOADING=1") # set up a pipe, fork off a child to handle cleanup later, # and rexec ourselves with an environment variable set which will # indicate which fd (one of the pipe ends) to write a byte to @@ -1041,6 +1046,9 @@ def run_wsgi(conf_path, app_section, *args, **kwargs): os.close(read_fd) except Exception: pass + else: + # SIGHUP or, less likely, run in "once" mode + systemd_notify(logger, "STOPPING=1") strategy.shutdown_sockets() signal.signal(signal.SIGTERM, signal.SIG_IGN) diff --git a/swift/container/updater.py b/swift/container/updater.py index 559618c066..6b2be12864 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -158,6 +158,7 @@ class ContainerUpdater(Daemon): pid2filename[pid] = tmpfilename else: signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.environ.pop('NOTIFY_SOCKET', None) eventlet_monkey_patch() self.no_changes = 0 self.successes = 0 diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 6f3e7f60d5..b34c3d7f32 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -368,6 +368,7 @@ class ObjectAuditor(Daemon): return pid else: signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.environ.pop('NOTIFY_SOCKET', None) if zero_byte_fps: kwargs['zero_byte_fps'] = self.conf_zero_byte_fps if sleep_between_zbf_scanner: diff --git a/swift/obj/updater.py b/swift/obj/updater.py index e75b6e7ad2..562c2847a1 100644 --- a/swift/obj/updater.py +++ b/swift/obj/updater.py @@ -381,6 +381,7 @@ class ObjectUpdater(Daemon): pids.append(pid) else: signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.environ.pop('NOTIFY_SOCKET', None) eventlet_monkey_patch() self.stats.reset() forkbegin = time.time() diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 0d022ab150..2de0daaf74 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -4052,7 +4052,16 @@ cluster_dfw1 = http://dfw1.host/v1/ m_socket.assert_called_once_with(socket.AF_UNIX, socket.SOCK_DGRAM) m_sock.connect.assert_called_once_with('foobar') m_sock.sendall.assert_called_once_with(b'READY=1') - self.assertNotIn('NOTIFY_SOCKET', os.environ) + # Still there, so we can send STOPPING/RELOADING messages + self.assertIn('NOTIFY_SOCKET', os.environ) + + m_socket.reset_mock() + m_sock.reset_mock() + logger = debug_logger() + utils.systemd_notify(logger, "RELOADING=1") + m_socket.assert_called_once_with(socket.AF_UNIX, socket.SOCK_DGRAM) + m_sock.connect.assert_called_once_with('foobar') + m_sock.sendall.assert_called_once_with(b'RELOADING=1') # Abstract notification socket m_socket.reset_mock() @@ -4062,7 +4071,7 @@ cluster_dfw1 = http://dfw1.host/v1/ m_socket.assert_called_once_with(socket.AF_UNIX, socket.SOCK_DGRAM) m_sock.connect.assert_called_once_with('\0foobar') m_sock.sendall.assert_called_once_with(b'READY=1') - self.assertNotIn('NOTIFY_SOCKET', os.environ) + self.assertIn('NOTIFY_SOCKET', os.environ) # Test logger with connection error m_sock = mock.Mock(connect=mock.Mock(side_effect=EnvironmentError), @@ -4094,7 +4103,7 @@ cluster_dfw1 = http://dfw1.host/v1/ msg = sock.recv(512) sock.close() self.assertEqual(msg, b'READY=1') - self.assertNotIn('NOTIFY_SOCKET', os.environ) + self.assertIn('NOTIFY_SOCKET', os.environ) # test file socket address socket_path = os.path.join(tempdir, 'foobar') diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index b72b78766c..e379dae748 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -976,6 +976,8 @@ class TestWSGI(unittest.TestCase, ConfigAssertMixin): mock.patch.object(wsgi, 'loadapp', _loadapp), \ mock.patch.object(wsgi, 'capture_stdio'), \ mock.patch.object(wsgi, 'run_server', _run_server), \ + mock.patch( + 'swift.common.wsgi.systemd_notify') as mock_notify, \ mock.patch('swift.common.utils.eventlet') as _utils_evt: wsgi.run_wsgi('conf_file', 'app_section', global_conf_callback=_global_conf_callback) @@ -986,6 +988,9 @@ class TestWSGI(unittest.TestCase, ConfigAssertMixin): socket=True, select=True, thread=True) + self.assertEqual(mock_notify.mock_calls, [ + mock.call('logger', "STOPPING=1"), + ]) def test_run_server_success(self): calls = defaultdict(int) @@ -1008,6 +1013,8 @@ class TestWSGI(unittest.TestCase, ConfigAssertMixin): mock.patch.object(wsgi, 'loadapp', _loadapp), \ mock.patch.object(wsgi, 'capture_stdio'), \ mock.patch.object(wsgi, 'run_server'), \ + mock.patch( + 'swift.common.wsgi.systemd_notify') as mock_notify, \ mock.patch('swift.common.utils.eventlet') as _utils_evt: rc = wsgi.run_wsgi('conf_file', 'app_section') self.assertEqual(calls['_initrp'], 1) @@ -1017,6 +1024,9 @@ class TestWSGI(unittest.TestCase, ConfigAssertMixin): socket=True, select=True, thread=True) + self.assertEqual(mock_notify.mock_calls, [ + mock.call('logger', "STOPPING=1"), + ]) # run_wsgi() no longer calls drop_privileges() in the parent process, # just clean_up_daemon_hygene() self.assertEqual([], _d_privs.mock_calls)