Merge "wsgi: Allow workers to gracefully exit"
This commit is contained in:
Zuul
Gerrit Code Review
commit
e6267a2bc1
|
|
@ -381,6 +381,17 @@ class Manager(object):
|
|||
status += 1
|
||||
return status
|
||||
|
||||
@command
|
||||
def kill_child_pids(self, **kwargs):
|
||||
"""kill child pids, optionally servicing accepted connections"""
|
||||
status = 0
|
||||
for server in self.servers:
|
||||
signaled_pids = server.kill_child_pids(**kwargs)
|
||||
if not signaled_pids:
|
||||
print(_('No %s running') % server)
|
||||
status += 1
|
||||
return status
|
||||
|
||||
@command
|
||||
def force_reload(self, **kwargs):
|
||||
"""alias for reload
|
||||
|
|
@ -594,6 +605,32 @@ class Server(object):
|
|||
pid = None
|
||||
yield pid_file, pid
|
||||
|
||||
def _signal_pid(self, sig, pid, pid_file, verbose):
|
||||
try:
|
||||
if sig != signal.SIG_DFL:
|
||||
print(_('Signal %(server)s pid: %(pid)s signal: '
|
||||
'%(signal)s') %
|
||||
{'server': self.server, 'pid': pid, 'signal': sig})
|
||||
safe_kill(pid, sig, 'swift-%s' % self.server)
|
||||
except InvalidPidFileException:
|
||||
if verbose:
|
||||
print(_('Removing pid file %(pid_file)s with wrong pid '
|
||||
'%(pid)d') % {'pid_file': pid_file, 'pid': pid})
|
||||
remove_file(pid_file)
|
||||
return False
|
||||
except OSError as e:
|
||||
if e.errno == errno.ESRCH:
|
||||
# pid does not exist
|
||||
if verbose:
|
||||
print(_("Removing stale pid file %s") % pid_file)
|
||||
remove_file(pid_file)
|
||||
elif e.errno == errno.EPERM:
|
||||
print(_("No permission to signal PID %d") % pid)
|
||||
return False
|
||||
else:
|
||||
# process exists
|
||||
return True
|
||||
|
||||
def signal_pids(self, sig, **kwargs):
|
||||
"""Send a signal to pids for this server
|
||||
|
||||
|
|
@ -608,30 +645,31 @@ class Server(object):
|
|||
print(_('Removing pid file %s with invalid pid') % pid_file)
|
||||
remove_file(pid_file)
|
||||
continue
|
||||
try:
|
||||
if sig != signal.SIG_DFL:
|
||||
print(_('Signal %(server)s pid: %(pid)s signal: '
|
||||
'%(signal)s') %
|
||||
{'server': self.server, 'pid': pid, 'signal': sig})
|
||||
safe_kill(pid, sig, 'swift-%s' % self.server)
|
||||
except InvalidPidFileException:
|
||||
if kwargs.get('verbose'):
|
||||
print(_('Removing pid file %(pid_file)s with wrong pid '
|
||||
'%(pid)d') % {'pid_file': pid_file, 'pid': pid})
|
||||
remove_file(pid_file)
|
||||
except OSError as e:
|
||||
if e.errno == errno.ESRCH:
|
||||
# pid does not exist
|
||||
if kwargs.get('verbose'):
|
||||
print(_("Removing stale pid file %s") % pid_file)
|
||||
remove_file(pid_file)
|
||||
elif e.errno == errno.EPERM:
|
||||
print(_("No permission to signal PID %d") % pid)
|
||||
else:
|
||||
# process exists
|
||||
if self._signal_pid(sig, pid, pid_file, kwargs.get('verbose')):
|
||||
pids[pid] = pid_file
|
||||
return pids
|
||||
|
||||
def signal_children(self, sig, **kwargs):
|
||||
"""Send a signal to child pids for this server
|
||||
|
||||
:param sig: signal to send
|
||||
|
||||
:returns: a dict mapping pids (ints) to pid_files (paths)
|
||||
|
||||
"""
|
||||
pids = {}
|
||||
for pid_file, pid in self.iter_pid_files(**kwargs):
|
||||
if not pid: # Catches None and 0
|
||||
print(_('Removing pid file %s with invalid pid') % pid_file)
|
||||
remove_file(pid_file)
|
||||
continue
|
||||
ps_cmd = ['ps', '--ppid', str(pid), '--no-headers', '-o', 'pid']
|
||||
for pid in subprocess.check_output(ps_cmd).split():
|
||||
pid = int(pid)
|
||||
if self._signal_pid(sig, pid, pid_file, kwargs.get('verbose')):
|
||||
pids[pid] = pid_file
|
||||
return pids
|
||||
|
||||
def get_running_pids(self, **kwargs):
|
||||
"""Get running pids
|
||||
|
||||
|
|
@ -659,6 +697,25 @@ class Server(object):
|
|||
sig = signal.SIGTERM
|
||||
return self.signal_pids(sig, **kwargs)
|
||||
|
||||
def kill_child_pids(self, **kwargs):
|
||||
"""Kill child pids, leaving server overseer to respawn them
|
||||
|
||||
:param graceful: if True, attempt SIGHUP on supporting servers
|
||||
:param seamless: if True, attempt SIGUSR1 on supporting servers
|
||||
|
||||
:returns: a dict mapping pids (ints) to pid_files (paths)
|
||||
|
||||
"""
|
||||
graceful = kwargs.get('graceful')
|
||||
seamless = kwargs.get('seamless')
|
||||
if graceful and self.server in GRACEFUL_SHUTDOWN_SERVERS:
|
||||
sig = signal.SIGHUP
|
||||
elif seamless and self.server in SEAMLESS_SHUTDOWN_SERVERS:
|
||||
sig = signal.SIGUSR1
|
||||
else:
|
||||
sig = signal.SIGTERM
|
||||
return self.signal_children(sig, **kwargs)
|
||||
|
||||
def status(self, pids=None, **kwargs):
|
||||
"""Display status of server
|
||||
|
||||
|
|
|
|||
|
|
@ -1111,9 +1111,13 @@ def run_wsgi(conf_path, app_section, *args, **kwargs):
|
|||
pid = os.fork()
|
||||
if pid == 0:
|
||||
os.close(read_fd)
|
||||
signal.signal(signal.SIGHUP, signal.SIG_DFL)
|
||||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||
signal.signal(signal.SIGUSR1, signal.SIG_DFL)
|
||||
|
||||
def shutdown_my_listen_sock(signum, *args):
|
||||
greenio.shutdown_safe(sock)
|
||||
|
||||
signal.signal(signal.SIGHUP, shutdown_my_listen_sock)
|
||||
signal.signal(signal.SIGUSR1, shutdown_my_listen_sock)
|
||||
strategy.post_fork_hook()
|
||||
|
||||
def notify():
|
||||
|
|
|
|||
|
|
@ -228,6 +228,42 @@ class SeamlessReloadMixin(object):
|
|||
self.manager.reload_seamless()
|
||||
|
||||
|
||||
class ChildReloadMixin(object):
|
||||
def make_post_reload_pid_cb(self):
|
||||
def _cb(post_reload_pids):
|
||||
# We expect all orig server PIDs to STILL BE PRESENT, no new server
|
||||
# present, and for there to be exactly 1 old worker PID plus
|
||||
# all but one additional new worker PIDs.
|
||||
num_workers = len(self.starting_pids['worker'])
|
||||
same_servers = (self.starting_pids['server'] ==
|
||||
post_reload_pids['server'])
|
||||
one_old_worker = 1 == len(self.starting_pids['worker'] &
|
||||
post_reload_pids['worker'])
|
||||
new_workers_present = (post_reload_pids['worker'] -
|
||||
self.starting_pids['worker'])
|
||||
return (post_reload_pids['server'] and same_servers and
|
||||
one_old_worker and
|
||||
len(new_workers_present) == num_workers - 1)
|
||||
return _cb
|
||||
|
||||
def make_post_close_pid_cb(self):
|
||||
def _cb(post_close_pids):
|
||||
# We expect all orig server PIDs to STILL BE PRESENT, no new server
|
||||
# present, no old worker PIDs, and all new worker PIDs.
|
||||
same_servers = (self.starting_pids['server'] ==
|
||||
post_close_pids['server'])
|
||||
old_workers_dead = not (self.starting_pids['worker'] &
|
||||
post_close_pids['worker'])
|
||||
new_workers_present = (post_close_pids['worker'] -
|
||||
self.starting_pids['worker'])
|
||||
return (post_close_pids['server'] and same_servers and
|
||||
old_workers_dead and new_workers_present)
|
||||
return _cb
|
||||
|
||||
def do_reload(self):
|
||||
self.manager.kill_child_pids(seamless=True)
|
||||
|
||||
|
||||
class TestObjectServerReloadBase(TestWSGIServerProcessHandling):
|
||||
SERVER_NAME = 'object'
|
||||
PID_TIMEOUT = 35
|
||||
|
|
@ -273,6 +309,14 @@ class TestObjectServerReloadSeamless(SeamlessReloadMixin,
|
|||
self._check_reload()
|
||||
|
||||
|
||||
class TestObjectServerReloadChild(ChildReloadMixin,
|
||||
TestObjectServerReloadBase):
|
||||
BODY = b'test-object' * 10
|
||||
|
||||
def test_object_reload_child(self):
|
||||
self._check_reload()
|
||||
|
||||
|
||||
class TestProxyServerReloadBase(TestWSGIServerProcessHandling):
|
||||
SERVER_NAME = 'proxy-server'
|
||||
HAS_INFO = True
|
||||
|
|
@ -358,6 +402,16 @@ class TestProxyServerReloadSeamless(SeamlessReloadMixin,
|
|||
self._check_reload()
|
||||
|
||||
|
||||
class TestProxyServerReloadChild(ChildReloadMixin,
|
||||
TestProxyServerReloadBase):
|
||||
BODY = b'proxy-seamless' * 10
|
||||
# A bit of a lie, but the respawned child won't pick up the updated config
|
||||
HAS_INFO = False
|
||||
|
||||
def test_proxy_reload_child(self):
|
||||
self._check_reload()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def spawn_services(ip_ports, timeout=10):
|
||||
q = eventlet.Queue()
|
||||
|
|
|
|||
Loading…
Reference in New Issue