680 lines
23 KiB
Python
680 lines
23 KiB
Python
# Copyright (c) 2010-2012 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import functools
|
|
import errno
|
|
import os
|
|
import resource
|
|
import signal
|
|
import time
|
|
import subprocess
|
|
import re
|
|
from swift import gettext_ as _
|
|
|
|
from swift.common.utils import search_tree, remove_file, write_file
|
|
|
|
SWIFT_DIR = '/etc/swift'
|
|
RUN_DIR = '/var/run/swift'
|
|
|
|
# auth-server has been removed from ALL_SERVERS, start it explicitly
|
|
ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor',
|
|
'container-replicator', 'container-reconciler',
|
|
'container-server', 'container-sync',
|
|
'container-updater', 'object-auditor', 'object-server',
|
|
'object-expirer', 'object-replicator',
|
|
'object-reconstructor', 'object-updater',
|
|
'proxy-server', 'account-replicator', 'account-reaper']
|
|
MAIN_SERVERS = ['proxy-server', 'account-server', 'container-server',
|
|
'object-server']
|
|
REST_SERVERS = [s for s in ALL_SERVERS if s not in MAIN_SERVERS]
|
|
GRACEFUL_SHUTDOWN_SERVERS = MAIN_SERVERS + ['auth-server']
|
|
START_ONCE_SERVERS = REST_SERVERS
|
|
# These are servers that match a type (account-*, container-*, object-*) but
|
|
# don't use that type-server.conf file and instead use their own.
|
|
STANDALONE_SERVERS = ['object-expirer', 'container-reconciler']
|
|
|
|
KILL_WAIT = 15 # seconds to wait for servers to die (by default)
|
|
WARNING_WAIT = 3 # seconds to wait after message that may just be a warning
|
|
|
|
MAX_DESCRIPTORS = 32768
|
|
MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB
|
|
MAX_PROCS = 8192 # workers * disks * threads_per_disk, can get high
|
|
|
|
|
|
def setup_env():
|
|
"""Try to increase resource limits of the OS. Move PYTHON_EGG_CACHE to /tmp
|
|
"""
|
|
try:
|
|
resource.setrlimit(resource.RLIMIT_NOFILE,
|
|
(MAX_DESCRIPTORS, MAX_DESCRIPTORS))
|
|
except ValueError:
|
|
print _("WARNING: Unable to modify file descriptor limit. "
|
|
"Running as non-root?")
|
|
|
|
try:
|
|
resource.setrlimit(resource.RLIMIT_DATA,
|
|
(MAX_MEMORY, MAX_MEMORY))
|
|
except ValueError:
|
|
print _("WARNING: Unable to modify memory limit. "
|
|
"Running as non-root?")
|
|
|
|
try:
|
|
resource.setrlimit(resource.RLIMIT_NPROC,
|
|
(MAX_PROCS, MAX_PROCS))
|
|
except ValueError:
|
|
print _("WARNING: Unable to modify max process limit. "
|
|
"Running as non-root?")
|
|
|
|
# Set PYTHON_EGG_CACHE if it isn't already set
|
|
os.environ.setdefault('PYTHON_EGG_CACHE', '/tmp')
|
|
|
|
|
|
def command(func):
|
|
"""
|
|
Decorator to declare which methods are accessible as commands, commands
|
|
always return 1 or 0, where 0 should indicate success.
|
|
|
|
:param func: function to make public
|
|
"""
|
|
func.publicly_accessible = True
|
|
|
|
@functools.wraps(func)
|
|
def wrapped(*a, **kw):
|
|
rv = func(*a, **kw)
|
|
return 1 if rv else 0
|
|
return wrapped
|
|
|
|
|
|
def watch_server_pids(server_pids, interval=1, **kwargs):
|
|
"""Monitor a collection of server pids yielding back those pids that
|
|
aren't responding to signals.
|
|
|
|
:param server_pids: a dict, lists of pids [int,...] keyed on
|
|
Server objects
|
|
"""
|
|
status = {}
|
|
start = time.time()
|
|
end = start + interval
|
|
server_pids = dict(server_pids) # make a copy
|
|
while True:
|
|
for server, pids in server_pids.items():
|
|
for pid in pids:
|
|
try:
|
|
# let pid stop if it wants to
|
|
os.waitpid(pid, os.WNOHANG)
|
|
except OSError as e:
|
|
if e.errno not in (errno.ECHILD, errno.ESRCH):
|
|
raise # else no such child/process
|
|
# check running pids for server
|
|
status[server] = server.get_running_pids(**kwargs)
|
|
for pid in pids:
|
|
# original pids no longer in running pids!
|
|
if pid not in status[server]:
|
|
yield server, pid
|
|
# update active pids list using running_pids
|
|
server_pids[server] = status[server]
|
|
if not [p for server, pids in status.items() for p in pids]:
|
|
# no more running pids
|
|
break
|
|
if time.time() > end:
|
|
break
|
|
else:
|
|
time.sleep(0.1)
|
|
|
|
|
|
class UnknownCommandError(Exception):
|
|
pass
|
|
|
|
|
|
class Manager(object):
|
|
"""Main class for performing commands on groups of servers.
|
|
|
|
:param servers: list of server names as strings
|
|
|
|
"""
|
|
|
|
def __init__(self, servers, run_dir=RUN_DIR):
|
|
self.server_names = set()
|
|
for server in servers:
|
|
if server == 'all':
|
|
self.server_names.update(ALL_SERVERS)
|
|
elif server == 'main':
|
|
self.server_names.update(MAIN_SERVERS)
|
|
elif server == 'rest':
|
|
self.server_names.update(REST_SERVERS)
|
|
elif '*' in server:
|
|
# convert glob to regex
|
|
self.server_names.update([
|
|
s for s in ALL_SERVERS if
|
|
re.match(server.replace('*', '.*'), s)])
|
|
else:
|
|
self.server_names.add(server)
|
|
|
|
self.servers = set()
|
|
for name in self.server_names:
|
|
self.servers.add(Server(name, run_dir))
|
|
|
|
def __iter__(self):
|
|
return iter(self.servers)
|
|
|
|
@command
|
|
def status(self, **kwargs):
|
|
"""display status of tracked pids for server
|
|
"""
|
|
status = 0
|
|
for server in self.servers:
|
|
status += server.status(**kwargs)
|
|
return status
|
|
|
|
@command
|
|
def start(self, **kwargs):
|
|
"""starts a server
|
|
"""
|
|
setup_env()
|
|
status = 0
|
|
|
|
for server in self.servers:
|
|
server.launch(**kwargs)
|
|
if not kwargs.get('daemon', True):
|
|
for server in self.servers:
|
|
try:
|
|
status += server.interact(**kwargs)
|
|
except KeyboardInterrupt:
|
|
print _('\nuser quit')
|
|
self.stop(**kwargs)
|
|
break
|
|
elif kwargs.get('wait', True):
|
|
for server in self.servers:
|
|
status += server.wait(**kwargs)
|
|
return status
|
|
|
|
@command
|
|
def no_wait(self, **kwargs):
|
|
"""spawn server and return immediately
|
|
"""
|
|
kwargs['wait'] = False
|
|
return self.start(**kwargs)
|
|
|
|
@command
|
|
def no_daemon(self, **kwargs):
|
|
"""start a server interactively
|
|
"""
|
|
kwargs['daemon'] = False
|
|
return self.start(**kwargs)
|
|
|
|
@command
|
|
def once(self, **kwargs):
|
|
"""start server and run one pass on supporting daemons
|
|
"""
|
|
kwargs['once'] = True
|
|
return self.start(**kwargs)
|
|
|
|
@command
|
|
def stop(self, **kwargs):
|
|
"""stops a server
|
|
"""
|
|
server_pids = {}
|
|
for server in self.servers:
|
|
signaled_pids = server.stop(**kwargs)
|
|
if not signaled_pids:
|
|
print _('No %s running') % server
|
|
else:
|
|
server_pids[server] = signaled_pids
|
|
|
|
# all signaled_pids, i.e. list(itertools.chain(*server_pids.values()))
|
|
signaled_pids = [p for server, pids in server_pids.items()
|
|
for p in pids]
|
|
# keep track of the pids yeiled back as killed for all servers
|
|
killed_pids = set()
|
|
kill_wait = kwargs.get('kill_wait', KILL_WAIT)
|
|
for server, killed_pid in watch_server_pids(server_pids,
|
|
interval=kill_wait,
|
|
**kwargs):
|
|
print _("%s (%s) appears to have stopped") % (server, killed_pid)
|
|
killed_pids.add(killed_pid)
|
|
if not killed_pids.symmetric_difference(signaled_pids):
|
|
# all processes have been stopped
|
|
return 0
|
|
|
|
# reached interval n watch_pids w/o killing all servers
|
|
for server, pids in server_pids.items():
|
|
if not killed_pids.issuperset(pids):
|
|
# some pids of this server were not killed
|
|
print _('Waited %s seconds for %s to die; giving up') % (
|
|
kill_wait, server)
|
|
return 1
|
|
|
|
@command
|
|
def kill(self, **kwargs):
|
|
"""stop a server (no error if not running)
|
|
"""
|
|
status = self.stop(**kwargs)
|
|
kwargs['quiet'] = True
|
|
if status and not self.status(**kwargs):
|
|
# only exit error if the server is still running
|
|
return status
|
|
return 0
|
|
|
|
@command
|
|
def shutdown(self, **kwargs):
|
|
"""allow current requests to finish on supporting servers
|
|
"""
|
|
kwargs['graceful'] = True
|
|
status = 0
|
|
status += self.stop(**kwargs)
|
|
return status
|
|
|
|
@command
|
|
def restart(self, **kwargs):
|
|
"""stops then restarts server
|
|
"""
|
|
status = 0
|
|
status += self.stop(**kwargs)
|
|
status += self.start(**kwargs)
|
|
return status
|
|
|
|
@command
|
|
def reload(self, **kwargs):
|
|
"""graceful shutdown then restart on supporting servers
|
|
"""
|
|
kwargs['graceful'] = True
|
|
status = 0
|
|
for server in self.server_names:
|
|
m = Manager([server])
|
|
status += m.stop(**kwargs)
|
|
status += m.start(**kwargs)
|
|
return status
|
|
|
|
@command
|
|
def force_reload(self, **kwargs):
|
|
"""alias for reload
|
|
"""
|
|
return self.reload(**kwargs)
|
|
|
|
def get_command(self, cmd):
|
|
"""Find and return the decorated method named like cmd
|
|
|
|
:param cmd: the command to get, a string, if not found raises
|
|
UnknownCommandError
|
|
|
|
"""
|
|
cmd = cmd.lower().replace('-', '_')
|
|
try:
|
|
f = getattr(self, cmd)
|
|
except AttributeError:
|
|
raise UnknownCommandError(cmd)
|
|
if not hasattr(f, 'publicly_accessible'):
|
|
raise UnknownCommandError(cmd)
|
|
return f
|
|
|
|
@classmethod
|
|
def list_commands(cls):
|
|
"""Get all publicly accessible commands
|
|
|
|
:returns: a list of string tuples (cmd, help), the method names who are
|
|
decorated as commands
|
|
"""
|
|
get_method = lambda cmd: getattr(cls, cmd)
|
|
return sorted([(x.replace('_', '-'), get_method(x).__doc__.strip())
|
|
for x in dir(cls) if
|
|
getattr(get_method(x), 'publicly_accessible', False)])
|
|
|
|
def run_command(self, cmd, **kwargs):
|
|
"""Find the named command and run it
|
|
|
|
:param cmd: the command name to run
|
|
|
|
"""
|
|
f = self.get_command(cmd)
|
|
return f(**kwargs)
|
|
|
|
|
|
class Server(object):
|
|
"""Manage operations on a server or group of servers of similar type
|
|
|
|
:param server: name of server
|
|
"""
|
|
|
|
def __init__(self, server, run_dir=RUN_DIR):
|
|
self.server = server.lower()
|
|
if '.' in self.server:
|
|
self.server, self.conf = self.server.rsplit('.', 1)
|
|
else:
|
|
self.conf = None
|
|
if '-' not in self.server:
|
|
self.server = '%s-server' % self.server
|
|
self.type = self.server.rsplit('-', 1)[0]
|
|
self.cmd = 'swift-%s' % self.server
|
|
self.procs = []
|
|
self.run_dir = run_dir
|
|
|
|
def __str__(self):
|
|
return self.server
|
|
|
|
def __repr__(self):
|
|
return "%s(%s)" % (self.__class__.__name__, repr(str(self)))
|
|
|
|
def __hash__(self):
|
|
return hash(str(self))
|
|
|
|
def __eq__(self, other):
|
|
try:
|
|
return self.server == other.server
|
|
except AttributeError:
|
|
return False
|
|
|
|
def get_pid_file_name(self, conf_file):
|
|
"""Translate conf_file to a corresponding pid_file
|
|
|
|
:param conf_file: an conf_file for this server, a string
|
|
|
|
:returns: the pid_file for this conf_file
|
|
|
|
"""
|
|
return conf_file.replace(
|
|
os.path.normpath(SWIFT_DIR), self.run_dir, 1).replace(
|
|
'%s-server' % self.type, self.server, 1).replace(
|
|
'.conf', '.pid', 1)
|
|
|
|
def get_conf_file_name(self, pid_file):
|
|
"""Translate pid_file to a corresponding conf_file
|
|
|
|
:param pid_file: a pid_file for this server, a string
|
|
|
|
:returns: the conf_file for this pid_file
|
|
|
|
"""
|
|
if self.server in STANDALONE_SERVERS:
|
|
return pid_file.replace(
|
|
os.path.normpath(self.run_dir), SWIFT_DIR, 1).replace(
|
|
'.pid', '.conf', 1)
|
|
else:
|
|
return pid_file.replace(
|
|
os.path.normpath(self.run_dir), SWIFT_DIR, 1).replace(
|
|
self.server, '%s-server' % self.type, 1).replace(
|
|
'.pid', '.conf', 1)
|
|
|
|
def conf_files(self, **kwargs):
|
|
"""Get conf files for this server
|
|
|
|
:param: number, if supplied will only lookup the nth server
|
|
|
|
:returns: list of conf files
|
|
"""
|
|
if self.server in STANDALONE_SERVERS:
|
|
server_search = self.server
|
|
else:
|
|
server_search = "%s-server" % self.type
|
|
if self.conf is not None:
|
|
found_conf_files = search_tree(SWIFT_DIR, server_search,
|
|
self.conf + '.conf',
|
|
dir_ext=self.conf + '.conf.d')
|
|
else:
|
|
found_conf_files = search_tree(SWIFT_DIR, server_search + '*',
|
|
'.conf', dir_ext='.conf.d')
|
|
number = kwargs.get('number')
|
|
if number:
|
|
try:
|
|
conf_files = [found_conf_files[number - 1]]
|
|
except IndexError:
|
|
conf_files = []
|
|
else:
|
|
conf_files = found_conf_files
|
|
if not conf_files:
|
|
# maybe there's a config file(s) out there, but I couldn't find it!
|
|
if not kwargs.get('quiet'):
|
|
print _('Unable to locate config %sfor %s') % (
|
|
('number %s ' % number if number else ''), self.server)
|
|
if kwargs.get('verbose') and not kwargs.get('quiet'):
|
|
if found_conf_files:
|
|
print _('Found configs:')
|
|
for i, conf_file in enumerate(found_conf_files):
|
|
print ' %d) %s' % (i + 1, conf_file)
|
|
|
|
return conf_files
|
|
|
|
def pid_files(self, **kwargs):
|
|
"""Get pid files for this server
|
|
|
|
:param: number, if supplied will only lookup the nth server
|
|
|
|
:returns: list of pid files
|
|
"""
|
|
if self.conf is not None:
|
|
pid_files = search_tree(self.run_dir, '%s*' % self.server,
|
|
exts=[self.conf + '.pid',
|
|
self.conf + '.pid.d'])
|
|
else:
|
|
pid_files = search_tree(self.run_dir, '%s*' % self.server)
|
|
if kwargs.get('number', 0):
|
|
conf_files = self.conf_files(**kwargs)
|
|
# filter pid_files to match the index of numbered conf_file
|
|
pid_files = [pid_file for pid_file in pid_files if
|
|
self.get_conf_file_name(pid_file) in conf_files]
|
|
return pid_files
|
|
|
|
def iter_pid_files(self, **kwargs):
|
|
"""Generator, yields (pid_file, pids)
|
|
"""
|
|
for pid_file in self.pid_files(**kwargs):
|
|
yield pid_file, int(open(pid_file).read().strip())
|
|
|
|
def signal_pids(self, sig, **kwargs):
|
|
"""Send a signal to pids for this server
|
|
|
|
:param sig: signal to send
|
|
|
|
:returns: a dict mapping pids (ints) to pid_files (paths)
|
|
|
|
"""
|
|
pids = {}
|
|
for pid_file, pid in self.iter_pid_files(**kwargs):
|
|
try:
|
|
if sig != signal.SIG_DFL:
|
|
print _('Signal %s pid: %s signal: %s') % (self.server,
|
|
pid, sig)
|
|
os.kill(pid, sig)
|
|
except OSError as e:
|
|
if e.errno == errno.ESRCH:
|
|
# pid does not exist
|
|
if kwargs.get('verbose'):
|
|
print _("Removing stale pid file %s") % pid_file
|
|
remove_file(pid_file)
|
|
elif e.errno == errno.EPERM:
|
|
print _("No permission to signal PID %d") % pid
|
|
else:
|
|
# process exists
|
|
pids[pid] = pid_file
|
|
return pids
|
|
|
|
def get_running_pids(self, **kwargs):
|
|
"""Get running pids
|
|
|
|
:returns: a dict mapping pids (ints) to pid_files (paths)
|
|
|
|
"""
|
|
return self.signal_pids(signal.SIG_DFL, **kwargs) # send noop
|
|
|
|
def kill_running_pids(self, **kwargs):
|
|
"""Kill running pids
|
|
|
|
:param graceful: if True, attempt SIGHUP on supporting servers
|
|
|
|
:returns: a dict mapping pids (ints) to pid_files (paths)
|
|
|
|
"""
|
|
graceful = kwargs.get('graceful')
|
|
if graceful and self.server in GRACEFUL_SHUTDOWN_SERVERS:
|
|
sig = signal.SIGHUP
|
|
else:
|
|
sig = signal.SIGTERM
|
|
return self.signal_pids(sig, **kwargs)
|
|
|
|
def status(self, pids=None, **kwargs):
|
|
"""Display status of server
|
|
|
|
:param: pids, if not supplied pids will be populated automatically
|
|
:param: number, if supplied will only lookup the nth server
|
|
|
|
:returns: 1 if server is not running, 0 otherwise
|
|
"""
|
|
if pids is None:
|
|
pids = self.get_running_pids(**kwargs)
|
|
if not pids:
|
|
number = kwargs.get('number', 0)
|
|
if number:
|
|
kwargs['quiet'] = True
|
|
conf_files = self.conf_files(**kwargs)
|
|
if conf_files:
|
|
print _("%s #%d not running (%s)") % (self.server, number,
|
|
conf_files[0])
|
|
else:
|
|
print _("No %s running") % self.server
|
|
return 1
|
|
for pid, pid_file in pids.items():
|
|
conf_file = self.get_conf_file_name(pid_file)
|
|
print _("%s running (%s - %s)") % (self.server, pid, conf_file)
|
|
return 0
|
|
|
|
def spawn(self, conf_file, once=False, wait=True, daemon=True, **kwargs):
|
|
"""Launch a subprocess for this server.
|
|
|
|
:param conf_file: path to conf_file to use as first arg
|
|
:param once: boolean, add once argument to command
|
|
:param wait: boolean, if true capture stdout with a pipe
|
|
:param daemon: boolean, if false ask server to log to console
|
|
|
|
:returns : the pid of the spawned process
|
|
"""
|
|
args = [self.cmd, conf_file]
|
|
if once:
|
|
args.append('once')
|
|
if not daemon:
|
|
# ask the server to log to console
|
|
args.append('verbose')
|
|
|
|
# figure out what we're going to do with stdio
|
|
if not daemon:
|
|
# do nothing, this process is open until the spawns close anyway
|
|
re_out = None
|
|
re_err = None
|
|
else:
|
|
re_err = subprocess.STDOUT
|
|
if wait:
|
|
# we're going to need to block on this...
|
|
re_out = subprocess.PIPE
|
|
else:
|
|
re_out = open(os.devnull, 'w+b')
|
|
proc = subprocess.Popen(args, stdout=re_out, stderr=re_err)
|
|
pid_file = self.get_pid_file_name(conf_file)
|
|
write_file(pid_file, proc.pid)
|
|
self.procs.append(proc)
|
|
return proc.pid
|
|
|
|
def wait(self, **kwargs):
|
|
"""
|
|
wait on spawned procs to start
|
|
"""
|
|
status = 0
|
|
for proc in self.procs:
|
|
# wait for process to close its stdout
|
|
output = proc.stdout.read()
|
|
if kwargs.get('once', False):
|
|
# if you don't want once to wait you can send it to the
|
|
# background on the command line, I generally just run with
|
|
# no-daemon anyway, but this is quieter
|
|
proc.wait()
|
|
if output:
|
|
print output
|
|
start = time.time()
|
|
# wait for process to die (output may just be a warning)
|
|
while time.time() - start < WARNING_WAIT:
|
|
time.sleep(0.1)
|
|
if proc.poll() is not None:
|
|
status += proc.returncode
|
|
break
|
|
return status
|
|
|
|
def interact(self, **kwargs):
|
|
"""
|
|
wait on spawned procs to terminate
|
|
"""
|
|
status = 0
|
|
for proc in self.procs:
|
|
# wait for process to terminate
|
|
proc.communicate()
|
|
if proc.returncode:
|
|
status += 1
|
|
return status
|
|
|
|
def launch(self, **kwargs):
|
|
"""
|
|
Collect conf files and attempt to spawn the processes for this server
|
|
"""
|
|
conf_files = self.conf_files(**kwargs)
|
|
if not conf_files:
|
|
return {}
|
|
|
|
pids = self.get_running_pids(**kwargs)
|
|
|
|
already_started = False
|
|
for pid, pid_file in pids.items():
|
|
conf_file = self.get_conf_file_name(pid_file)
|
|
# for legacy compat you can't start other servers if one server is
|
|
# already running (unless -n specifies which one you want), this
|
|
# restriction could potentially be lifted, and launch could start
|
|
# any unstarted instances
|
|
if conf_file in conf_files:
|
|
already_started = True
|
|
print _("%s running (%s - %s)") % (self.server, pid, conf_file)
|
|
elif not kwargs.get('number', 0):
|
|
already_started = True
|
|
print _("%s running (%s - %s)") % (self.server, pid, pid_file)
|
|
|
|
if already_started:
|
|
print _("%s already started...") % self.server
|
|
return {}
|
|
|
|
if self.server not in START_ONCE_SERVERS:
|
|
kwargs['once'] = False
|
|
|
|
pids = {}
|
|
for conf_file in conf_files:
|
|
if kwargs.get('once'):
|
|
msg = _('Running %s once') % self.server
|
|
else:
|
|
msg = _('Starting %s') % self.server
|
|
print '%s...(%s)' % (msg, conf_file)
|
|
try:
|
|
pid = self.spawn(conf_file, **kwargs)
|
|
except OSError as e:
|
|
if e.errno == errno.ENOENT:
|
|
#TODO(clayg): should I check if self.cmd exists earlier?
|
|
print _("%s does not exist") % self.cmd
|
|
break
|
|
else:
|
|
raise
|
|
pids[pid] = conf_file
|
|
|
|
return pids
|
|
|
|
def stop(self, **kwargs):
|
|
"""Send stop signals to pids for this server
|
|
|
|
:returns: a dict mapping pids (ints) to pid_files (paths)
|
|
|
|
"""
|
|
return self.kill_running_pids(**kwargs)
|