From 0b380264d76fff5031e75c2c8245ae5add10f221 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 17 Nov 2010 17:52:31 -0600 Subject: [PATCH 01/64] refactored swift-init to be more maintainable, added some features, help, docs, tests, fixed lp:639710; new bin/test module with examples of how to import files from bin and some stubs for swift-init --- .bintests | 2 + bin/swift-init | 776 ++++++++++++++++++++++++++------ doc/source/development_saio.rst | 16 +- swift/common/utils.py | 6 +- test/bin/__init__.py | 32 ++ test/bin/test_swift_init.py | 468 +++++++++++++++++++ 6 files changed, 1146 insertions(+), 154 deletions(-) create mode 100644 .bintests create mode 100644 test/bin/__init__.py create mode 100644 test/bin/test_swift_init.py diff --git a/.bintests b/.bintests new file mode 100644 index 0000000000..55a94b303a --- /dev/null +++ b/.bintests @@ -0,0 +1,2 @@ +#!/bin/bash +nosetests test/bin -v diff --git a/bin/swift-init b/bin/swift-init index cd31ea8a93..5d45e1e891 100755 --- a/bin/swift-init +++ b/bin/swift-init @@ -15,6 +15,7 @@ # limitations under the License. from __future__ import with_statement +import functools import errno import glob import os @@ -22,169 +23,668 @@ import resource import signal import sys import time +from optparse import OptionParser +import subprocess + +from swift.common import utils + +SWIFT_DIR = '/etc/swift' +RUN_DIR = '/var/run/swift' ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor', 'container-replicator', 'container-server', 'container-updater', 'object-auditor', 'object-server', 'object-replicator', 'object-updater', 'proxy-server', 'account-replicator', 'auth-server', 'account-reaper'] -GRACEFUL_SHUTDOWN_SERVERS = ['account-server', 'container-server', - 'object-server', 'proxy-server', 'auth-server'] +MAIN_SERVERS = ['auth-server', 'proxy-server', 'account-server', + 'container-server', 'object-server'] +REST_SERVERS = [s for s in ALL_SERVERS if s not in MAIN_SERVERS] +GRACEFUL_SHUTDOWN_SERVERS = MAIN_SERVERS +START_ONCE_SERVERS = REST_SERVERS + +KILL_WAIT = 15 # seconds to wait for servers to die + MAX_DESCRIPTORS = 32768 -MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB +MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB -_, server, command = sys.argv -if server == 'all': - servers = ALL_SERVERS -else: - if '-' not in server: - server = '%s-server' % server - servers = [server] -command = command.lower() - -def pid_files(server): - if os.path.exists('/var/run/swift/%s.pid' % server): - pid_files = ['/var/run/swift/%s.pid' % server] - else: - pid_files = glob.glob('/var/run/swift/%s/*.pid' % server) - for pid_file in pid_files: - pid = int(open(pid_file).read().strip()) - yield pid_file, pid - -def do_start(server, once=False): - server_type = '-'.join(server.split('-')[:-1]) - - for pid_file, pid in pid_files(server): - if os.path.exists('/proc/%s' % pid): - print "%s appears to already be running: %s" % (server, pid_file) - return - else: - print "Removing stale pid file %s" % pid_file - os.unlink(pid_file) +def setup_env(): + """Try to increase resource limits of the OS. Move PYTHON_EGG_CACHE to /tmp + """ try: resource.setrlimit(resource.RLIMIT_NOFILE, (MAX_DESCRIPTORS, MAX_DESCRIPTORS)) resource.setrlimit(resource.RLIMIT_DATA, (MAX_MEMORY, MAX_MEMORY)) except ValueError: - print "Unable to increase file descriptor limit. Running as non-root?" + print "WARNING: Unable to increase file descriptor limit." \ + " Running as non-root?" + os.environ['PYTHON_EGG_CACHE'] = '/tmp' + return - def write_pid_file(pid_file, pid): - dir, file = os.path.split(pid_file) - if not os.path.exists(dir): - try: - os.makedirs(dir) - except OSError, err: - if err.errno == errno.EACCES: - sys.exit('Unable to create %s. Running as non-root?' % dir) - fp = open(pid_file, 'w') - fp.write('%d\n' % pid) - fp.close() - def launch(ini_file, pid_file): - cmd = 'swift-%s' % server - args = [server, ini_file] - if once: - print 'Running %s once' % server - args.append('once') +def search_tree(root, glob_match, ext): + """Look in root, for any files/dirs matching glob, recurively traversing + any found directories looking for files ending with ext + + :param root: start of search path + :param glob_match: glob to match in root, matching dirs are traversed with + os.walk + :param ext: only files that end in ext will be returned + + :returns: list of full paths to matching files, sorted + + """ + found_files = [] + for path in glob.glob(os.path.join(root, glob_match)): + if path.endswith(ext): + found_files.append(path) else: - print 'Starting %s' % server + for root, dirs, files in os.walk(path): + for file in files: + if file.endswith(ext): + found_files.append(os.path.join(root, file)) + return sorted(found_files) - pid = os.fork() - if pid == 0: - os.setsid() - with open(os.devnull, 'r+b') as nullfile: - for desc in (0, 1, 2): # close stdio + +def write_file(path, contents): + """Write contents to file at path + + :param path: any path, subdirs will be created as needed + :param contents: data to write to file, will be converted to string + + """ + dir, name = os.path.split(path) + if not os.path.exists(dir): + try: + os.makedirs(dir) + except OSError, err: + if err.errno == errno.EACCES: + sys.exit('Unable to create %s. Running as non-root?' % + dir) + with open(path, 'w') as f: + f.write('%s' % contents) + + +def remove_file(path): + """Quiet wrapper for os.unlink, OSErrors are suppressed + + :param path: first and only argument passed to os.unlink + """ + try: + os.unlink(path) + except OSError: + pass + + +def command(func): + """ + Decorator to declare which methods are accessible as commands, commands + always return 1 or 0, where 0 should indicate success. + + :param func: function to make public + """ + func.publicly_accessible = True + + @functools.wraps(func) + def wrapped(*a, **kw): + rv = func(*a, **kw) + return 1 if rv else 0 + return wrapped + + +class UnknownCommand(Exception): + pass + + +class SwiftInit(): + """Main class for performing commands on groups of servers. + + :param servers: list of server names as strings + + """ + + def __init__(self, servers): + self.servers = [] + + server_names = set() + for server in servers: + if server == 'all': + server_names.update(ALL_SERVERS) + elif server == 'main': + server_names.update(MAIN_SERVERS) + elif server == 'rest': + server_names.update(REST_SERVERS) + else: + server_names.add(server) + + for name in server_names: + self.servers.append(SwiftServer(name)) + + def watch_server_pids(self, server_pids, interval=0, **kwargs): + """Monitor a collection of server pids yeilding back those pids that + aren't responding to signals. + + :param server_pids: a dict, lists of pids [int,...] keyed on + SwiftServer objects + """ + status = {} + start = time.time() + end = start + interval + while interval: + for server, pids in server_pids.items(): + for pid in pids: + # let pid stop if it wants to try: - os.dup2(nullfile.fileno(), desc) - except OSError: - pass - try: - if once: - os.execlp('swift-%s' % server, server, - ini_file, 'once') - else: - os.execlp('swift-%s' % server, server, ini_file) - except OSError: - print 'unable to launch %s' % server - sys.exit(0) - else: - write_pid_file(pid_file, pid) - - ini_file = '/etc/swift/%s-server.conf' % server_type - if os.path.exists(ini_file): - # single config file over-rides config dirs - pid_file = '/var/run/swift/%s.pid' % server - launch_args = [(ini_file, pid_file)] - elif os.path.exists('/etc/swift/%s-server/' % server_type): - # found config directory, searching for config file(s) - launch_args = [] - for num, ini_file in enumerate(glob.glob('/etc/swift/%s-server/*.conf' % server_type)): - pid_file = '/var/run/swift/%s/%d.pid' % (server, num) - # start a server for each ini_file found - launch_args.append((ini_file, pid_file)) - else: - # maybe there's a config file(s) out there, but I couldn't find it! - sys.exit('Unable to locate config file for %s. %s does not exist?' % (server, ini_file)) - - # start all servers - for ini_file, pid_file in launch_args: - launch(ini_file, pid_file) - -def do_stop(server, graceful=False): - if graceful and server in GRACEFUL_SHUTDOWN_SERVERS: - sig = signal.SIGHUP - else: - sig = signal.SIGTERM - - did_anything = False - pfiles = pid_files(server) - for pid_file, pid in pfiles: - did_anything = True - try: - print 'Stopping %s pid: %s signal: %s' % (server, pid, sig) - os.kill(pid, sig) - except OSError: - print "Process %d not running" % pid - try: - os.unlink(pid_file) - except OSError: - pass - for pid_file, pid in pfiles: - for _ in xrange(150): # 15 seconds - if not os.path.exists('/proc/%s' % pid): + os.waitpid(pid, os.WNOHANG) + except OSError, e: + if e.errno in (errno.ECHILD, errno.ESRCH): + pass + else: + raise + status[server] = server.get_running_pids(**kwargs) + #print server, pids, status[server] + for pid in pids: + if pid not in status[server]: + yield server, pid + server_pids[server] = status[server] + if not [p for server, pids in status.items() for p in pids]: + # no more running pids break - time.sleep(0.1) + if time.time() > end: + break + else: + time.sleep(0.1) + return + + @command + def status(self, **kwargs): + """display status of tracked pids for server + """ + status = 0 + for server in self.servers: + status += server.status(**kwargs) + return status + + @command + def start(self, **kwargs): + """starts a server + """ + setup_env() + status = 0 + + for server in self.servers: + server.launch(**kwargs) + if kwargs.get('wait', False): + for server in self.servers: + status += server.wait(**kwargs) + if not kwargs.get('daemon', True): + for server in self.servers: + try: + status += server.interact(**kwargs) + except KeyboardInterrupt: + print '\nuser quit' + self.stop(**kwargs) + break + return status + + @command + def wait(self, daemon=True, **kwargs): + """spawn server and wait for it to start + """ + kwargs['wait'] = True + self.start(**kwargs) + + @command + def no_daemon(self, **kwargs): + """start a server interactivly + """ + kwargs['daemon'] = False + self.start(**kwargs) + + @command + def once(self, **kwargs): + """start server and run one pass on supporting daemons + """ + kwargs['once'] = True + return self.start(**kwargs) + + @command + def stop(self, **kwargs): + """stops a server + """ + server_pids = {} + for server in self.servers: + signaled_pids = server.stop(**kwargs) + if not signaled_pids: + print 'No %s running' % server + else: + server_pids[server] = signaled_pids + + # all signaled_pids, i.e. list(itertools.chain(*server_pids.values())) + signaled_pids = [p for server, pid in server_pids.items() for p in pid] + # keep track of the pids yeiled back as killed for all servers + killed_pids = set() + for server, killed_pid in self.watch_server_pids(server_pids, + interval=KILL_WAIT, **kwargs): + print "%s (%s) appears to have stopped" % (server, killed_pid) + killed_pids.add(killed_pid) + if not killed_pids.symmetric_difference(signaled_pids): + # all proccesses have been stopped + return 0 + + # reached interval n watch_pids w/o killing all servers + for server, pids in server_pids.items(): + if not killed_pids.issuperset(pids): + # some pids of this server were not killed + print 'Waited 15 seconds for %s to die; giving up' % (server) + return 1 + + @command + def shutdown(self, **kwargs): + """allow current requests to finish on supporting servers + """ + kwargs['graceful'] = True + status = 0 + self.stop(**kwargs) + return status + + @command + def restart(self, **kwargs): + """stops then restarts server + """ + status = 0 + self.stop(**kwargs) + self.start(**kwargs) + return status + + @command + def reload(self, **kwargs): + """graceful shutdown then restart on supporting servers + """ + kwargs['graceful'] = True + status = 0 + for server in self.servers: + init = SwiftInit([server.server]) + status += init.stop(**kwargs) + status += init.start(**kwargs) + return status + + @command + def force_reload(self, **kwargs): + """alias for reload + """ + return self.reload(**kwargs) + + def get_command(self, cmd): + """Find and return the decorated method named like cmd + + :param cmd: the command to get, a string, if not found raises + UnknownCommand + + """ + cmd = cmd.lower().replace('-', '_') + try: + f = getattr(self, cmd) + except AttributeError: + raise UnknownCommand(cmd) + if not hasattr(f, 'publicly_accessible'): + raise UnknownCommand(cmd) + return f + + def list_commands(self): + """Get all publicly accessible commands + + :returns: a list of strings, the method names who are decorated + as commands + """ + cmds = sorted([x.replace('_', '-') for x in dir(self) if \ + hasattr(getattr(self, x), 'publicly_accessible')]) + try: + helps = [self.get_command(x).__doc__.strip() for x in cmds] + except AttributeError: + raise AttributeError( + 'command %s has no __doc__, please add one' % x) + return zip(cmds, helps) + + def run_command(self, cmd, **kwargs): + """Find the named command and run it + + :param cmd: the command name to run + + """ + f = self.get_command(cmd) + return f(**kwargs) + + +class SwiftServer(): + """Manage operations on a server or group of servers of similar type + + :param server: name of server + """ + + def __init__(self, server): + if '-' not in server: + server = '%s-server' % server + self.server = server + self.type = '-'.join(server.split('-')[:-1]) + self.cmd = 'swift-%s' % server + self.procs = [] + + def __str__(self): + return self.server + + def get_pid_file_name(self, ini_file): + """Translate ini_file to a corresponding pid_file + + :param ini_file: an ini_file for this server, a string + + :returns: the pid_file for this ini_file + + """ + return ini_file.replace( + os.path.normpath(SWIFT_DIR), RUN_DIR, 1).replace( + '%s-server' % self.type, self.server, 1).rsplit( + '.conf', 1)[0] + '.pid' + + def get_ini_file_name(self, pid_file): + """Translate pid_file to a corresponding ini_file + + :param pid_file: a pid_file for this server, a string + + :returns: the ini_file for this pid_file + + """ + return pid_file.replace( + os.path.normpath(RUN_DIR), SWIFT_DIR, 1).replace( + self.server, '%s-server' % self.type, 1).rsplit( + '.pid', 1)[0] + '.conf' + + def ini_files(self, **kwargs): + """Get ini files for this server + + :param: number, if supplied will only lookup the nth server + + :returns: list of ini files + """ + found_ini_files = search_tree(SWIFT_DIR, '%s-server*' % self.type, + '.conf') + number = kwargs.get('number') + if number: + try: + ini_files = [found_ini_files[number - 1]] + except IndexError: + ini_files = [] else: - print 'Waited 15 seconds for pid %s (%s) to die; giving up' % \ - (pid, pid_file) - if not did_anything: - print 'No %s running' % server + ini_files = found_ini_files + if not ini_files: + # maybe there's a config file(s) out there, but I couldn't find it! + if not kwargs.get('quiet'): + print('Unable to locate config %sfor %s' % ( + ('number %s ' % number if number else ''), self.server)) + if kwargs.get('verbose') and not kwargs.get('quiet'): + if found_ini_files: + print('Found configs:') + for i, ini_file in enumerate(found_ini_files): + print(' %d) %s' % (i + 1, ini_file)) -if command == 'start': - for server in servers: - do_start(server) + return ini_files -if command == 'stop': - for server in servers: - do_stop(server) + def pid_files(self, **kwargs): + """Get pid files for this server -if command == 'shutdown': - for server in servers: - do_stop(server, graceful=True) + :param: number, if supplied will only lookup the nth server -if command == 'restart': - for server in servers: - do_stop(server) - for server in servers: - do_start(server) + :returns: list of pid files + """ + pid_files = search_tree(RUN_DIR, '%s*' % self.server, '.pid') + number = kwargs.get('number', 0) + if number: + ini_files = self.ini_files(**kwargs) + # limt pid_files the one who translates to the indexed ini_file for + # this given number + pid_files = [pid_file for pid_file in pid_files if + self.get_ini_file_name(pid_file) in ini_files] + return pid_files -if command == 'reload' or command == 'force-reload': - for server in servers: - do_stop(server, graceful=True) - do_start(server) + def iter_pid_files(self, **kwargs): + """Generator, yields (pid_file, pids) + """ + for pid_file in self.pid_files(**kwargs): + yield pid_file, int(open(pid_file).read().strip()) -if command == 'once': - for server in servers: - do_start(server, once=True) + def signal_pids(self, sig, **kwargs): + """Send a signal to pids for this server + + :param sig: signal to send + + :returns: a dict mapping pids (ints) to pid_files (paths) + + """ + pids = {} + for pid_file, pid in self.iter_pid_files(**kwargs): + try: + if sig != signal.SIG_DFL: + print 'Signal %s pid: %s signal: %s' % ( + self.server, pid, sig) + os.kill(pid, sig) + except OSError, e: + #print '%s sig err: %s' % (pid, e) + if e.errno == 3: + # pid does not exist + if kwargs.get('verbose'): + print "Removing stale pid file %s" % pid_file + remove_file(pid_file) + else: + # process exists + pids[pid] = pid_file + return pids + + def get_running_pids(self, **kwargs): + """Get running pids + + :returns: a dict mapping pids (ints) to pid_files (paths) + + """ + return self.signal_pids(signal.SIG_DFL, **kwargs) # send noop + + def kill_running_pids(self, **kwargs): + """Get running pids + + :param graceful: if True, attempt SIGHUP on supporting servers + + :returns: a dict mapping pids (ints) to pid_files (paths) + + """ + graceful = kwargs.get('graceful') + if graceful and self.server in GRACEFUL_SHUTDOWN_SERVERS: + sig = signal.SIGHUP + else: + sig = signal.SIGTERM + return self.signal_pids(sig, **kwargs) + + def status(self, pids=None, **kwargs): + """Display status of server + + :param: pids, if not supplied pids will be populated automatically + :param: number, if supplied will only lookup the nth server + + :returns: 1 if server is not running, 0 otherwise + """ + if pids is None: + pids = self.get_running_pids(**kwargs) + if not pids: + number = kwargs.get('number', 0) + if number: + kwargs['quiet'] = True + ini_files = self.ini_files(**kwargs) + if ini_files: + print "%s #%d not running (%s)" % (self.server, number, + ini_files[0]) + else: + print "No %s running" % self.server + return 1 + for pid, pid_file in pids.items(): + ini_file = self.get_ini_file_name(pid_file) + print "%s running (%s - %s)" % (self.server, pid, ini_file) + return 0 + + def spawn(self, ini_file, once=False, wait=False, daemon=True, **kwargs): + """Launch a subprocess for this server. + + :param ini_file: path to ini_file to use as first arg + :param once: boolean, add once argument to command + :param wait: boolean, if true capture stdout with a pipe + :param daemon: boolean, if true ask server to log to console + """ + args = [self.cmd, ini_file] + if once: + args.append('once') + if not daemon: + # ask the server to log to console + args.append('verbose') + + # figure out what we're going to do with stdio + if not daemon: + # do nothing, this process is open until the spawns close anyway + re_out = None + re_err = None + else: + re_err = subprocess.STDOUT + if wait: + # we're going to need to block on this... + re_out = subprocess.PIPE + else: + re_out = open(os.devnull, 'w+b') + proc = subprocess.Popen(args, stdout=re_out, stderr=re_err) + pid_file = self.get_pid_file_name(ini_file) + write_file(pid_file, proc.pid) + self.procs.append(proc) + + def wait(self, **kwargs): + """ + wait on spawned procs to start + """ + status = 0 + for proc in self.procs: + # wait for process to close it's stdout + output = proc.stdout.read() + if output: + print output + if proc.returncode: + status += 1 + return status + + def interact(self, **kwargs): + """ + wait on spawned procs to terminate + """ + status = 0 + for proc in self.procs: + # wait for process to terminate + proc.communicate()[0] + if proc.returncode: + status += 1 + return status + + def launch(self, **kwargs): + """ + Collect ini files and attempt to spawn the processes for this server + """ + ini_files = self.ini_files(**kwargs) + if not ini_files: + return [] + + pids = self.get_running_pids(**kwargs) + + already_started = False + for pid, pid_file in pids.items(): + ini_file = self.get_ini_file_name(pid_file) + # for legacy compat you can't start other servers if one server is + # already running (unless -n specifies which one you want), this + # restriction could potentially be lifted, and launch could start + # any unstarted instances + if ini_file in ini_files or not number: + already_started = True + print "%s running (%s - %s)" % (self.server, pid, ini_file) + + if already_started: + print "%s already started..." % self.server + #self.status(pids) + return [] + + if self.server not in START_ONCE_SERVERS: + kwargs['once'] = False + + pids = {} + for ini_file in ini_files: + if kwargs.get('once'): + msg = 'Running %s once' % self.server + else: + msg = 'Starting %s' % self.server + print '%s...(%s)' % (msg, ini_file) + pid = self.spawn(ini_file, **kwargs) + pids[pid] = ini_file + + return pids + + def stop(self, **kwargs): + """Send stop signals to pids for this server + + :returns: a dict mapping pids (ints) to pid_files (paths) + + """ + return self.kill_running_pids(**kwargs) + + +USAGE = """%prog [ + +Commands: +""" + '\n'.join(["%16s: %s" % x for x in SwiftInit([]).list_commands()]) + + +def main(): + parser = OptionParser(USAGE) + parser.add_option('-v', '--verbose', action="store_true", + default=False, help="display verbose output") + parser.add_option('-w', '--wait', action="store_true", default=False, + help="wait for server to start before returning") + parser.add_option('-o', '--once', action="store_true", + default=False, help="only run one pass of daemon") + # this is a negative option, default is options.daemon = True + parser.add_option('-n', '--no-daemon', action="store_false", dest="daemon", + default=True, help="start server interactively") + parser.add_option('-g', '--graceful', action="store_true", + default=False, help="send SIGHUP to supporting servers") + parser.add_option('-c', '--config-num', metavar="N", type="int", + dest="number", default=0, + help="send command to the Nth server only") + options, args = parser.parse_args() + + if len(args) < 2: + parser.print_help() + print 'ERROR: specify server(s) and command' + return 1 + + command = args[-1] + servers = args[:-1] + + if len(servers) == 1: + # this is just a stupid swap for me cause I always try to "start main" + commands, docs = zip(*SwiftInit([]).list_commands()) + if servers[0] in commands: + command, servers = servers[0], [command] + + init = SwiftInit(servers) + try: + status = init.run_command(command, **options.__dict__) + except UnknownCommand: + parser.print_help() + print 'ERROR: unknown command, %s' % command + return 1 + + return 1 if status else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index a270753338..9988a18f7d 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -561,11 +561,7 @@ Setting up scripts for running Swift #!/bin/bash - swift-init auth-server start - swift-init proxy-server start - swift-init account-server start - swift-init container-server start - swift-init object-server start + swift-init main start #. Create `~/bin/startrest`:: @@ -574,15 +570,7 @@ Setting up scripts for running Swift # Replace devauth with whatever your super_admin key is (recorded in # /etc/swift/auth-server.conf). swift-auth-recreate-accounts -K devauth - swift-init object-updater start - swift-init container-updater start - swift-init object-replicator start - swift-init container-replicator start - swift-init account-replicator start - swift-init object-auditor start - swift-init container-auditor start - swift-init account-auditor start - swift-init account-reaper start + swift-init rest start #. `chmod +x ~/bin/*` #. `remakerings` diff --git a/swift/common/utils.py b/swift/common/utils.py index 9311c28fce..78b8870f48 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -71,8 +71,10 @@ TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes', 'on', 'On')) def validate_configuration(): if HASH_PATH_SUFFIX == '': - sys.exit("Error: [swift-hash]: swift_hash_path_suffix missing " - "from /etc/swift/swift.conf") + print "Error: [swift-hash]: swift_hash_path_suffix missing from " \ + "/etc/swift/swift.conf" + sys.exit(1) + def load_libc_function(func_name): diff --git a/test/bin/__init__.py b/test/bin/__init__.py new file mode 100644 index 0000000000..4d50c67b05 --- /dev/null +++ b/test/bin/__init__.py @@ -0,0 +1,32 @@ +'''Tests for packaged "bin" scripts +''' + +import os +import imp # access the import internals + +bin_test_dir = os.path.dirname(__file__) + +bin_dir = os.path.join(bin_test_dir, '../../bin') + +# build up logical mapping of bin 'file-name' to (module, path) +modules = {} +for file_name in os.listdir(bin_dir): + module = '.'.join(['test', 'bin', file_name.replace('-', '_')]) + path = os.path.join(bin_dir, file_name) + modules[file_name] = (module, path) + +def get_bin_module(bin_file_name): + name, path = modules[bin_file_name] + try: + module = imp.load_source(name, path) + finally: + # another option would be adding bin/*c to .bzrignore + try: + os.unlink(path + 'c') + except OSError: + pass + return module + +# this might not be safe on source files that don't test __name__=="__main__" +swift_init = get_bin_module('swift-init') +st = get_bin_module('st') diff --git a/test/bin/test_swift_init.py b/test/bin/test_swift_init.py new file mode 100644 index 0000000000..190afb1603 --- /dev/null +++ b/test/bin/test_swift_init.py @@ -0,0 +1,468 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import os +import signal +from shutil import rmtree +from contextlib import contextmanager +from tempfile import mkdtemp +from collections import defaultdict + +from test import bin # for reloading... +from test.bin import swift_init # for testing... + + +CONF_FILES = ( + 'auth-server.conf', + 'auth-server.pid', + 'proxy-server/proxy-server.conf', + 'proxy-server.conf', + 'proxy.conf', + 'object-server/1.conf', + 'object-server/2.conf', + 'object-server/3.conf', + 'object-server/4.conf', + 'account-server/a1.conf', + 'account-server/a2.conf', + 'account-server/a3.conf', + 'container-server/1/container-server.conf', + 'container-server/2/container-server.conf', + 'container-server/3/container-server.conf', + 'container-server/4/container-server.conf', + 'container-server/5/container-server.conf', + 'container-replicator/bogus.conf', +) + +PID_FILES = ( + ('auth-server.pid', 10), + ('proxy-server/proxy-server.pid', 21), + ('proxy-server.pid', 22), + ('object-server/1.pid', 31), + ('object-server/2.pid', 32), + ('object-server/3.pid', 33), + ('object-server/4.pid', 34), + ('object-auditor/2.pid', 42), + ('object-auditor/1.pid', 41), + ('object-auditor/3.pid', 43), + ('container-replicator/1/container-server.pid', 51), + ('container-replicator/2/container-server.pid', 52), + ('container-replicator/3/container-server.pid', 53), + ('container-replicator/4/container-server.pid', 54), + ('container-replicator/5/container-server.pid', 55), +) + +# maps all servers to a list of running pids as defined in PID_FILES +PID_MAP = defaultdict(list) +for server in swift_init.ALL_SERVERS: + for pid_file, pid in PID_FILES: + if pid_file.startswith(server) and pid_file.endswith('.pid'): + PID_MAP[server].append(pid) + PID_MAP[server].sort() +ALL_PIDS = [p for server, pids in PID_MAP.items() for p in pids] + +DUMMY_SIG = 1 + + +@contextmanager +def temptree(files, contents=''): + # generate enough contents to fill the files + c = len(files) + contents = (list(contents) + [''] * c)[:c] + tempdir = mkdtemp() + for path, content in zip(files, contents): + if os.path.isabs(path): + path = '.' + path + new_path = os.path.join(tempdir, path) + subdir = os.path.dirname(new_path) + if not os.path.exists(subdir): + os.makedirs(subdir) + with open(new_path, 'w') as f: + f.write(str(content)) + try: + yield tempdir + finally: + rmtree(tempdir) + + +class MockOs(): + + def __init__(self, pids): + self.pid_sigs = defaultdict(list) + for pid in pids: + self.pid_sigs[pid] + self.closed_fds = [] + self.child_pid = 9999 # fork defaults to test parent process path + self.execlp_called = False + + def kill(self, pid, sig): + if pid not in self.pid_sigs: + raise OSError(3, 'No such process') + self.pid_sigs[pid].append(sig) + + """ + def pass_func(self, *args, **kwargs): + pass + + chdir = setsid = umask = pass_func + + def dup2(self, source, target): + self.closed_fds.append(target) + + def fork(self): + return self.child_pid + + def execlp(self, *args): + self.execlp_called = True + + """ + + def __getattr__(self, name): + # I only over-ride portions of the os module + try: + return object.__getattr__(self, name) + except AttributeError: + return getattr(os, name) + + +class TestSwiftInitModule(unittest.TestCase): + + def test_servers(self): + main_plus_rest = set(swift_init.MAIN_SERVERS + swift_init.REST_SERVERS) + self.assertEquals(set(swift_init.ALL_SERVERS), main_plus_rest) + # make sure there's no server listed in both + self.assertEquals(len(main_plus_rest), len(swift_init.MAIN_SERVERS) + + len(swift_init.REST_SERVERS)) + + def test_search_tree(self): + with temptree(CONF_FILES) as t: + auth_conf = swift_init.search_tree(t, 'auth-server*', '.conf') + self.assertEquals(len(auth_conf), 1) + self.assertEquals(auth_conf[0], + os.path.join(t, 'auth-server.conf')) + proxy_conf = swift_init.search_tree(t, 'proxy*', 'conf') + self.assertEquals(len(proxy_conf), 3) + self.assertEquals(proxy_conf[0], + os.path.join(t, 'proxy-server.conf')) + object_confs = swift_init.search_tree(t, 'object-server*', '.conf') + self.assertEquals(len(object_confs), 4) + for i, conf in enumerate(object_confs): + path = os.path.join(t, 'object-server/%d.conf' % (i + 1)) + self.assertEquals(conf, path) + account_confs = swift_init.search_tree(t, 'account-server*', + '.conf') + self.assertEquals(len(account_confs), 3) + for i, conf in enumerate(account_confs): + path = os.path.join(t, 'account-server/a%d.conf' % (i + 1)) + self.assertEquals(conf, path) + container_confs = swift_init.search_tree(t, 'container-server*', + '.conf') + self.assertEquals(len(container_confs), 5) + for i, conf in enumerate(container_confs): + path = os.path.join(t, + 'container-server/%s/container-server.conf' % (i + 1)) + self.assertEquals(conf, path) + + def test_write_file(self): + with temptree([]) as t: + file_name = os.path.join(t, 'test') + swift_init.write_file(file_name, 'test') + with open(file_name, 'r') as f: + contents = f.read() + self.assertEquals(contents, 'test') + + def test_remove_file(self): + with temptree([]) as t: + file_name = os.path.join(t, 'blah.pid') + # assert no raise + self.assertEquals(os.path.exists(file_name), False) + self.assertEquals(swift_init.remove_file(file_name), None) + with open(file_name, 'w') as f: + f.write('1') + self.assert_(os.path.exists(file_name)) + self.assertEquals(swift_init.remove_file(file_name), None) + self.assertFalse(os.path.exists(file_name)) + + def test_command_wrapper(self): + @swift_init.command + def myfunc(arg1): + """test doc + """ + return arg1 + + self.assertEquals(myfunc.__doc__.strip(), 'test doc') + self.assertEquals(myfunc(1), 1) + self.assertEquals(myfunc(0), 0) + self.assertEquals(myfunc(True), 1) + self.assertEquals(myfunc(False), 0) + self.assert_(hasattr(myfunc, 'publicly_accessible')) + self.assert_(myfunc.publicly_accessible) + + def test_exc(self): + self.assert_(issubclass(swift_init.UnknownCommand, Exception)) + + +class TestSwiftServerClass(unittest.TestCase): + + def tearDown(self): + reload(bin) + + def join_swift_dir(self, path): + return os.path.join(swift_init.SWIFT_DIR, path) + + def join_run_dir(self, path): + return os.path.join(swift_init.RUN_DIR, path) + + def test_server_init(self): + server = swift_init.SwiftServer('proxy') + self.assertEquals(server.server, 'proxy-server') + self.assertEquals(server.type, 'proxy') + self.assertEquals(server.cmd, 'swift-proxy-server') + server = swift_init.SwiftServer('object-replicator') + self.assertEquals(server.server, 'object-replicator') + self.assertEquals(server.type, 'object') + self.assertEquals(server.cmd, 'swift-object-replicator') + + def test_get_pid_file_name(self): + server = swift_init.SwiftServer('proxy') + ini_file = self.join_swift_dir('proxy-server.conf') + pid_file = self.join_run_dir('proxy-server.pid') + self.assertEquals(pid_file, server.get_pid_file_name(ini_file)) + server = swift_init.SwiftServer('object-replicator') + ini_file = self.join_swift_dir('object-server/1.conf') + pid_file = self.join_run_dir('object-replicator/1.pid') + self.assertEquals(pid_file, server.get_pid_file_name(ini_file)) + server = swift_init.SwiftServer('container-auditor') + ini_file = self.join_swift_dir( + 'container-server/1/container-auditor.conf') + pid_file = self.join_run_dir( + 'container-auditor/1/container-auditor.pid') + self.assertEquals(pid_file, server.get_pid_file_name(ini_file)) + + def test_get_ini_file_name(self): + server = swift_init.SwiftServer('proxy') + ini_file = self.join_swift_dir('proxy-server.conf') + pid_file = self.join_run_dir('proxy-server.pid') + self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) + server = swift_init.SwiftServer('object-replicator') + ini_file = self.join_swift_dir('object-server/1.conf') + pid_file = self.join_run_dir('object-replicator/1.pid') + self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) + server = swift_init.SwiftServer('container-auditor') + ini_file = self.join_swift_dir( + 'container-server/1/container-auditor.conf') + pid_file = self.join_run_dir( + 'container-auditor/1/container-auditor.pid') + self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) + + def test_ini_files(self): + with temptree(CONF_FILES) as t: + swift_init.SWIFT_DIR = t + servers = \ + 'auth object account-auditor container-replicator'.split() + for server in servers: + server = swift_init.SwiftServer(server) + ini_files = server.ini_files() + conf_files = [self.join_swift_dir(x) for x in CONF_FILES if + x.startswith('%s-server' % server.type) and + x.endswith('.conf')] + self.assertEquals(ini_files, conf_files) + + # test config number kwarg returns the correct single config + for i in range(len(ini_files)): + ini_files = server.ini_files(number=i + 1) + self.assertEquals(len(ini_files), 1) + self.assertEquals(ini_files[0], conf_files[i]) + + # and returns empty for missing config number + ini_files = server.ini_files(number=i + 2) + self.assertEquals(ini_files, []) + + + def test_pid_files(self): + files, contents = zip(*PID_FILES) + # throw some garbage files in the mix too + files += CONF_FILES + files += tuple([f.replace('.conf', '.bogus') for f in CONF_FILES]) + + def assert_pid_files(server): + server = swift_init.SwiftServer(server) + pid_files = server.pid_files() + # should only find pids files + self.assertEquals(len(pid_files), len(PID_MAP[server.server])) + real_files = sorted([x for x in files if x.startswith(server.server) and + x.endswith('.pid')]) + for i, pid_file in enumerate(pid_files): + self.assertEquals(pid_file, self.join_run_dir(real_files[i])) + + # test config number kwarg returns the correct single pid file + for i in range(len(pid_files)): + ini_files = server.ini_files(number=i + 1) + pid_files = server.pid_files(number=i + 1) + self.assertEquals(len(pid_files), 1) + pid_file = pid_files[0] + self.assertEquals(pid_file, self.join_run_dir(real_files[i])) + + with temptree(files, contents) as t: + swift_init.RUN_DIR = t + swift_init.SWIFT_DIR = t + servers = \ + 'auth proxy object object-auditor container-replicator'.split() + for server in servers: + assert_pid_files(server) + + def test_iter_pid_files(self): + with temptree(*zip(*PID_FILES)) as t: + swift_init.RUN_DIR = t + servers = \ + 'auth proxy object object-auditor container-replicator'.split() + # build up a mapping of pid_files to pids for these server types + pid_file_map = {} + for name, pid in PID_FILES: + if [server for server in servers if server in name]: + pid_file_map[self.join_run_dir(name)] = pid + for i, server in enumerate(servers): + server = swift_init.SwiftServer(server) + for pid_file, pid in server.iter_pid_files(): + self.assertEquals(pid, pid_file_map[pid_file]) + del pid_file_map[pid_file] + self.assert_(pid_file_map == {}) # all pid_files used up + + def test_signal_pids(self): + # stale pid file cleanup is tested in get_running_pids + def assert_server_signaled_pids(server): + server = swift_init.SwiftServer(server) + signaled_pids = server.signal_pids(DUMMY_SIG) + self.assertEquals(sorted(signaled_pids.keys()), PID_MAP[server.server]) + for pid in signaled_pids: + self.assertEquals(swift_init.os.pid_sigs[pid], [1]) + + with temptree(*zip(*PID_FILES)) as t: + swift_init.RUN_DIR = t + swift_init.os = MockOs(ALL_PIDS) + servers = \ + 'auth proxy object object-auditor container-replicator'.split() + for server in servers: + assert_server_signaled_pids(server) + + def test_get_running_pids(self): + def assert_running_pids(server): + # all pids running to start + swift_init.os = MockOs(ALL_PIDS) + server = swift_init.SwiftServer(server) + running_pids = server.get_running_pids() + # make sure we only get pids for this server + self.assertEquals(sorted(running_pids.keys()), PID_MAP[server.server]) + for pid in running_pids: + # verify the correct signal on the mock + self.assertEquals(swift_init.os.pid_sigs[pid], + [signal.SIG_DFL]) + current_pids = list(PID_MAP[server.server]) + # each pass will assert the pid from the previous run was removed + last_pid = -1 # this pid is skipped on the first pass + for pid in list(current_pids): + # setup current running pids + swift_init.os = MockOs(current_pids) + if not (last_pid == -1): + # any valid pid should show up in iter_pid_files + self.assert_(last_pid in [ + p for f, p in list(server.iter_pid_files())]) + running_pids = server.get_running_pids() + self.assert_(pid in running_pids) + self.assertFalse(last_pid in running_pids) + # assert stale pid files were removed! + self.assertEquals(len(running_pids), len(server.pid_files())) + if last_pid in ALL_PIDS: + self.assertFalse(last_pid in [ + p for f, p in list(server.iter_pid_files())]) + # get ready for next iter, simulate pid died (i.e. remove # pid) + last_pid = pid + current_pids.remove(pid) + self.assertEquals(len(current_pids), 0) + swift_init.os = MockOs(current_pids) + self.assertEquals(len(server.get_running_pids()), 0) + + with temptree(*zip(*PID_FILES)) as t: + swift_init.RUN_DIR = t + servers = \ + 'auth proxy object object-auditor container-replicator'.split() + for server in servers: + assert_running_pids(server) + + def test_kill_running_pids(self): + def assert_kill_running_pids(server): + server = swift_init.SwiftServer(server) + # all pids running + swift_init.os = MockOs(ALL_PIDS) + + pid_files = server.pid_files() + killed_pids = server.kill_running_pids() + self.assertEquals(len(killed_pids), len(pid_files)) + + # reinit mock os + swift_init.os = MockOs(ALL_PIDS) + # make sure graceful sends sighup on servers that support it + huped_pids = server.kill_running_pids(graceful=True) + if server.server in swift_init.GRACEFUL_SHUTDOWN_SERVERS: + captured_sig = signal.SIGHUP + else: + captured_sig = signal.SIGTERM + for pid in huped_pids: + self.assertEquals(swift_init.os.pid_sigs[pid], [captured_sig]) + + with temptree(*zip(*PID_FILES)) as t: + swift_init.RUN_DIR = t + servers = \ + 'auth proxy object object-auditor container-replicator'.split() + for server in servers: + assert_kill_running_pids(server) + + #TODO: more tests + def test_status(self): + pass + + def test_spawn(self): + pass + + def test_wait(self): + pass + + def test_interact(self): + pass + + def test_launch(self): + pass + + def test_stop(self): + pass + + +#TODO: test SwiftInit class +class TestSwiftInitClass(unittest.TestCase): + + def test_placeholder(self): + pass + + +#TODO: test main +class TestMain(unittest.TestCase): + + def test_placeholder(self): + pass + + +if __name__ == '__main__': + unittest.main() From 73210937a37505d85a0688ba7b7012301a0f20b3 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 18 Nov 2010 16:47:44 -0600 Subject: [PATCH 02/64] made existing tests more literal/readable/useful --- .bintests | 0 bin/swift-init | 2 +- test/bin/test_swift_init.py | 619 ++++++++++++++++++++++-------------- 3 files changed, 379 insertions(+), 242 deletions(-) mode change 100644 => 100755 .bintests diff --git a/.bintests b/.bintests old mode 100644 new mode 100755 diff --git a/bin/swift-init b/bin/swift-init index 5d45e1e891..b537ba8341 100755 --- a/bin/swift-init +++ b/bin/swift-init @@ -489,7 +489,7 @@ class SwiftServer(): return self.signal_pids(signal.SIG_DFL, **kwargs) # send noop def kill_running_pids(self, **kwargs): - """Get running pids + """Kill running pids :param graceful: if True, attempt SIGHUP on supporting servers diff --git a/test/bin/test_swift_init.py b/test/bin/test_swift_init.py index 190afb1603..37082c4eb2 100644 --- a/test/bin/test_swift_init.py +++ b/test/bin/test_swift_init.py @@ -16,6 +16,7 @@ import unittest import os +import sys import signal from shutil import rmtree from contextlib import contextmanager @@ -25,58 +26,8 @@ from collections import defaultdict from test import bin # for reloading... from test.bin import swift_init # for testing... - -CONF_FILES = ( - 'auth-server.conf', - 'auth-server.pid', - 'proxy-server/proxy-server.conf', - 'proxy-server.conf', - 'proxy.conf', - 'object-server/1.conf', - 'object-server/2.conf', - 'object-server/3.conf', - 'object-server/4.conf', - 'account-server/a1.conf', - 'account-server/a2.conf', - 'account-server/a3.conf', - 'container-server/1/container-server.conf', - 'container-server/2/container-server.conf', - 'container-server/3/container-server.conf', - 'container-server/4/container-server.conf', - 'container-server/5/container-server.conf', - 'container-replicator/bogus.conf', -) - -PID_FILES = ( - ('auth-server.pid', 10), - ('proxy-server/proxy-server.pid', 21), - ('proxy-server.pid', 22), - ('object-server/1.pid', 31), - ('object-server/2.pid', 32), - ('object-server/3.pid', 33), - ('object-server/4.pid', 34), - ('object-auditor/2.pid', 42), - ('object-auditor/1.pid', 41), - ('object-auditor/3.pid', 43), - ('container-replicator/1/container-server.pid', 51), - ('container-replicator/2/container-server.pid', 52), - ('container-replicator/3/container-server.pid', 53), - ('container-replicator/4/container-server.pid', 54), - ('container-replicator/5/container-server.pid', 55), -) - -# maps all servers to a list of running pids as defined in PID_FILES -PID_MAP = defaultdict(list) -for server in swift_init.ALL_SERVERS: - for pid_file, pid in PID_FILES: - if pid_file.startswith(server) and pid_file.endswith('.pid'): - PID_MAP[server].append(pid) - PID_MAP[server].sort() -ALL_PIDS = [p for server, pids in PID_MAP.items() for p in pids] - DUMMY_SIG = 1 - @contextmanager def temptree(files, contents=''): # generate enough contents to fill the files @@ -101,35 +52,17 @@ def temptree(files, contents=''): class MockOs(): def __init__(self, pids): + self.running_pids = pids self.pid_sigs = defaultdict(list) - for pid in pids: - self.pid_sigs[pid] self.closed_fds = [] self.child_pid = 9999 # fork defaults to test parent process path self.execlp_called = False def kill(self, pid, sig): - if pid not in self.pid_sigs: + if pid not in self.running_pids: raise OSError(3, 'No such process') self.pid_sigs[pid].append(sig) - """ - def pass_func(self, *args, **kwargs): - pass - - chdir = setsid = umask = pass_func - - def dup2(self, source, target): - self.closed_fds.append(target) - - def fork(self): - return self.child_pid - - def execlp(self, *args): - self.execlp_called = True - - """ - def __getattr__(self, name): # I only over-ride portions of the os module try: @@ -138,6 +71,17 @@ class MockOs(): return getattr(os, name) +def pop_stream(f): + """read everything out of file from the top and clear it out + """ + f.flush() + f.seek(0) + output = f.read() + f.seek(0) + f.truncate() + print >> sys.stderr, output + return output + class TestSwiftInitModule(unittest.TestCase): def test_servers(self): @@ -148,33 +92,54 @@ class TestSwiftInitModule(unittest.TestCase): len(swift_init.REST_SERVERS)) def test_search_tree(self): - with temptree(CONF_FILES) as t: - auth_conf = swift_init.search_tree(t, 'auth-server*', '.conf') - self.assertEquals(len(auth_conf), 1) - self.assertEquals(auth_conf[0], - os.path.join(t, 'auth-server.conf')) - proxy_conf = swift_init.search_tree(t, 'proxy*', 'conf') - self.assertEquals(len(proxy_conf), 3) - self.assertEquals(proxy_conf[0], - os.path.join(t, 'proxy-server.conf')) - object_confs = swift_init.search_tree(t, 'object-server*', '.conf') - self.assertEquals(len(object_confs), 4) - for i, conf in enumerate(object_confs): - path = os.path.join(t, 'object-server/%d.conf' % (i + 1)) - self.assertEquals(conf, path) - account_confs = swift_init.search_tree(t, 'account-server*', - '.conf') - self.assertEquals(len(account_confs), 3) - for i, conf in enumerate(account_confs): - path = os.path.join(t, 'account-server/a%d.conf' % (i + 1)) - self.assertEquals(conf, path) - container_confs = swift_init.search_tree(t, 'container-server*', - '.conf') - self.assertEquals(len(container_confs), 5) - for i, conf in enumerate(container_confs): - path = os.path.join(t, - 'container-server/%s/container-server.conf' % (i + 1)) - self.assertEquals(conf, path) + # file match & ext miss + with temptree(['asdf.conf', 'blarg.conf', 'asdf.cfg']) as t: + asdf = swift_init.search_tree(t, 'a*', '.conf') + self.assertEquals(len(asdf), 1) + self.assertEquals(asdf[0], + os.path.join(t, 'asdf.conf')) + + # multi-file match & glob miss & sort + with temptree(['application.bin', 'apple.bin', 'apropos.bin']) as t: + app_bins = swift_init.search_tree(t, 'app*', 'bin') + self.assertEquals(len(app_bins), 2) + self.assertEquals(app_bins[0], + os.path.join(t, 'apple.bin')) + self.assertEquals(app_bins[1], + os.path.join(t, 'application.bin')) + + # test file in folder & ext miss & glob miss + files = ( + 'sub/file1.ini', + 'sub/file2.conf', + 'sub.bin', + 'bus.ini', + 'bus/file3.ini', + ) + with temptree(files) as t: + sub_ini = swift_init.search_tree(t, 'sub*', '.ini') + self.assertEquals(len(sub_ini), 1) + self.assertEquals(sub_ini[0], + os.path.join(t, 'sub/file1.ini')) + + # test multi-file in folder & sub-folder & ext miss & glob miss + files = ( + 'folder_file.txt', + 'folder/1.txt', + 'folder/sub/2.txt', + 'folder2/3.txt', + 'Folder3/4.txt' + 'folder.rc', + ) + with temptree(files) as t: + folder_texts = swift_init.search_tree(t, 'folder*', '.txt') + self.assertEquals(len(folder_texts), 4) + f1 = os.path.join(t, 'folder_file.txt') + f2 = os.path.join(t, 'folder/1.txt') + f3 = os.path.join(t, 'folder/sub/2.txt') + f4 = os.path.join(t, 'folder2/3.txt') + for f in [f1, f2, f3, f4]: + self.assert_(f in folder_texts) def test_write_file(self): with temptree([]) as t: @@ -269,166 +234,338 @@ class TestSwiftServerClass(unittest.TestCase): self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) def test_ini_files(self): - with temptree(CONF_FILES) as t: + # test get single ini file + ini_files = ( + 'proxy-server.conf', + 'proxy-server.ini', + 'auth-server.conf', + ) + with temptree(ini_files) as t: swift_init.SWIFT_DIR = t - servers = \ - 'auth object account-auditor container-replicator'.split() - for server in servers: - server = swift_init.SwiftServer(server) - ini_files = server.ini_files() - conf_files = [self.join_swift_dir(x) for x in CONF_FILES if - x.startswith('%s-server' % server.type) and - x.endswith('.conf')] - self.assertEquals(ini_files, conf_files) + server = swift_init.SwiftServer('proxy') + ini_files = server.ini_files() + self.assertEquals(len(ini_files), 1) + ini_file = ini_files[0] + self.assertEquals(ini_file, self.join_swift_dir('proxy-server.conf')) - # test config number kwarg returns the correct single config - for i in range(len(ini_files)): - ini_files = server.ini_files(number=i + 1) - self.assertEquals(len(ini_files), 1) - self.assertEquals(ini_files[0], conf_files[i]) - - # and returns empty for missing config number - ini_files = server.ini_files(number=i + 2) - self.assertEquals(ini_files, []) - - - def test_pid_files(self): - files, contents = zip(*PID_FILES) - # throw some garbage files in the mix too - files += CONF_FILES - files += tuple([f.replace('.conf', '.bogus') for f in CONF_FILES]) - - def assert_pid_files(server): - server = swift_init.SwiftServer(server) - pid_files = server.pid_files() - # should only find pids files - self.assertEquals(len(pid_files), len(PID_MAP[server.server])) - real_files = sorted([x for x in files if x.startswith(server.server) and - x.endswith('.pid')]) - for i, pid_file in enumerate(pid_files): - self.assertEquals(pid_file, self.join_run_dir(real_files[i])) - - # test config number kwarg returns the correct single pid file - for i in range(len(pid_files)): - ini_files = server.ini_files(number=i + 1) - pid_files = server.pid_files(number=i + 1) - self.assertEquals(len(pid_files), 1) - pid_file = pid_files[0] - self.assertEquals(pid_file, self.join_run_dir(real_files[i])) - - with temptree(files, contents) as t: - swift_init.RUN_DIR = t + # test multi server conf files & grouping of server-type config + ini_files = ( + 'object-server1.conf', + 'object-server/2.conf', + 'object-server/object3.conf', + 'object-server/conf/server4.conf', + 'object-server.txt', + 'proxy-server.conf', + ) + with temptree(ini_files) as t: swift_init.SWIFT_DIR = t - servers = \ - 'auth proxy object object-auditor container-replicator'.split() - for server in servers: - assert_pid_files(server) + server = swift_init.SwiftServer('object-replicator') + ini_files = server.ini_files() + self.assertEquals(len(ini_files), 4) + c1 = self.join_swift_dir('object-server1.conf') + c2 = self.join_swift_dir('object-server/2.conf') + c3 = self.join_swift_dir('object-server/object3.conf') + c4 = self.join_swift_dir('object-server/conf/server4.conf') + for c in [c1, c2, c3, c4]: + self.assert_(c in ini_files) + # test configs returned sorted + sorted_confs = sorted([c1, c2, c3, c4]) + self.assertEquals(ini_files, sorted_confs) + + + # test get single numbered conf + ini_files = ( + 'account-server/1.conf', + 'account-server/2.conf', + 'account-server/3.conf', + 'account-server/4.conf', + ) + with temptree(ini_files) as t: + swift_init.SWIFT_DIR = t + server = swift_init.SwiftServer('account') + ini_files = server.ini_files(number=2) + self.assertEquals(len(ini_files), 1) + ini_file = ini_files[0] + self.assertEquals(ini_file, + self.join_swift_dir('account-server/2.conf')) + # test missing config number + ini_files = server.ini_files(number=5) + self.assertFalse(ini_files) + + # test verbose & quiet + ini_files = ( + 'auth-server.ini', + 'container-server/1.conf', + ) + with temptree(ini_files) as t: + swift_init.SWIFT_DIR = t + old_stdout = sys.stdout + try: + with open('std.out', 'w+') as f: + sys.stdout = f + server = swift_init.SwiftServer('auth') + # check warn "unable to locate" + ini_files = server.ini_files() + self.assertFalse(ini_files) + self.assert_('unable to locate' in pop_stream(f).lower()) + # check quiet will silence warning + ini_files = server.ini_files(verbose=True, quiet=True) + self.assertEquals(pop_stream(f), '') + # check found config no warning + server = swift_init.SwiftServer('container-auditor') + ini_files = server.ini_files() + self.assertEquals(pop_stream(f), '') + # check missing config number warn "unable to locate" + ini_files = server.ini_files(number=2) + self.assert_('unable to locate' in pop_stream(f).lower()) + # check verbose lists configs + ini_files = server.ini_files(number=2, verbose=True) + c1 = self.join_swift_dir('container-server/1.conf') + self.assert_(c1 in pop_stream(f)) + finally: + sys.stdout = old_stdout def test_iter_pid_files(self): - with temptree(*zip(*PID_FILES)) as t: + """ + SwiftServer.iter_pid_files is kinda boring, test the + SwiftServer.pid_files stuff here as well + """ + pid_files = ( + ('proxy-server.pid', 1), + ('auth-server.pid', 'blah'), + ('object-replicator/1.pid', 11), + ('object-replicator/2.pid', 12), + ) + files, contents = zip(*pid_files) + with temptree(files, contents) as t: swift_init.RUN_DIR = t - servers = \ - 'auth proxy object object-auditor container-replicator'.split() - # build up a mapping of pid_files to pids for these server types - pid_file_map = {} - for name, pid in PID_FILES: - if [server for server in servers if server in name]: - pid_file_map[self.join_run_dir(name)] = pid - for i, server in enumerate(servers): - server = swift_init.SwiftServer(server) + server = swift_init.SwiftServer('proxy') + # test get one file + iter = server.iter_pid_files() + pid_file, pid = iter.next() + self.assertEquals(pid_file, self.join_run_dir('proxy-server.pid')) + self.assertEquals(pid, 1) + # ... and only one file + self.assertRaises(StopIteration, iter.next) + # test invalid value in pid file + server = swift_init.SwiftServer('auth') + self.assertRaises(ValueError, server.iter_pid_files().next) + # test object-server doesn't steal pids from object-replicator + server = swift_init.SwiftServer('object') + self.assertRaises(StopIteration, server.iter_pid_files().next) + # test multi-pid iter + server = swift_init.SwiftServer('object-replicator') + real_map = { + 11: self.join_run_dir('object-replicator/1.pid'), + 12: self.join_run_dir('object-replicator/2.pid'), + } + pid_map = {} + for pid_file, pid in server.iter_pid_files(): + pid_map[pid] = pid_file + self.assertEquals(pid_map, real_map) + + # test get pid_files by number + ini_files = ( + 'object-server/1.conf', + 'object-server/2.conf', + 'object-server/3.conf', + 'object-server/4.conf', + ) + + pid_files = ( + ('object-server/1.pid', 1), + ('object-server/2.pid', 2), + ('object-server/5.pid', 5), + ) + + with temptree(ini_files) as swift_dir: + swift_init.SWIFT_DIR = swift_dir + files, pids = zip(*pid_files) + with temptree(files, pids) as t: + swift_init.RUN_DIR = t + server = swift_init.SwiftServer('object') + # test get all pid files + real_map = { + 1: self.join_run_dir('object-server/1.pid'), + 2: self.join_run_dir('object-server/2.pid'), + 5: self.join_run_dir('object-server/5.pid'), + } + pid_map = {} for pid_file, pid in server.iter_pid_files(): - self.assertEquals(pid, pid_file_map[pid_file]) - del pid_file_map[pid_file] - self.assert_(pid_file_map == {}) # all pid_files used up + pid_map[pid] = pid_file + self.assertEquals(pid_map, real_map) + # test get pid with matching conf + pids = list(server.iter_pid_files(number=2)) + self.assertEquals(len(pids), 1) + pid_file, pid = pids[0] + self.assertEquals(pid, 2) + self.assertEquals(pid_file, self.join_run_dir('object-server/2.pid')) + # try to iter on a pid number with a matching conf but no pid + pids = list(server.iter_pid_files(number=3)) + self.assertFalse(pids) + # test get pids w/o matching conf + pids = list(server.iter_pid_files(number=5)) + self.assertFalse(pids) def test_signal_pids(self): - # stale pid file cleanup is tested in get_running_pids - def assert_server_signaled_pids(server): - server = swift_init.SwiftServer(server) - signaled_pids = server.signal_pids(DUMMY_SIG) - self.assertEquals(sorted(signaled_pids.keys()), PID_MAP[server.server]) - for pid in signaled_pids: - self.assertEquals(swift_init.os.pid_sigs[pid], [1]) - - with temptree(*zip(*PID_FILES)) as t: + pid_files = ( + ('proxy-server.pid', 1), + ('auth-server.pid', 2), + ) + files, pids = zip(*pid_files) + with temptree(files, pids) as t: swift_init.RUN_DIR = t - swift_init.os = MockOs(ALL_PIDS) - servers = \ - 'auth proxy object object-auditor container-replicator'.split() - for server in servers: - assert_server_signaled_pids(server) + # mock os with both pids running + swift_init.os = MockOs([1, 2]) + server = swift_init.SwiftServer('proxy') + pids = server.signal_pids(DUMMY_SIG) + self.assertEquals(len(pids), 1) + self.assert_(1 in pids) + self.assertEquals(swift_init.os.pid_sigs[1], [DUMMY_SIG]) + # make sure other process not signaled + self.assertFalse(2 in pids) + self.assertFalse(2 in swift_init.os.pid_sigs) + # capture stdio + old_stdout = sys.stdout + try: + with open('std.out', 'w+') as f: + sys.stdout = f + #test print details + pids = server.signal_pids(DUMMY_SIG) + output = pop_stream(f) + self.assert_('pid: %s' % 1 in output) + self.assert_('signal: %s' % DUMMY_SIG in output) + # test no details on signal.SIG_DFL + pids = server.signal_pids(signal.SIG_DFL) + self.assertEquals(pop_stream(f), '') + # reset mock os so only the other server is running + swift_init.os = MockOs([2]) + # test pid not running + pids = server.signal_pids(signal.SIG_DFL) + self.assert_(1 not in pids) + self.assert_(1 not in swift_init.os.pid_sigs) + # test remove stale pid file + self.assertFalse(os.path.exists( + self.join_run_dir('proxy-server.pid'))) + # reset mock os with no running pids + swift_init.os = MockOs([]) + server = swift_init.SwiftServer('auth') + # test verbose warns on removing pid file + pids = server.signal_pids(signal.SIG_DFL, verbose=True) + output = pop_stream(f) + self.assert_('stale pid' in output.lower()) + self.assert_(self.join_run_dir('auth-server.pid') in output) + finally: + sys.stdout = old_stdout def test_get_running_pids(self): - def assert_running_pids(server): - # all pids running to start - swift_init.os = MockOs(ALL_PIDS) - server = swift_init.SwiftServer(server) - running_pids = server.get_running_pids() - # make sure we only get pids for this server - self.assertEquals(sorted(running_pids.keys()), PID_MAP[server.server]) - for pid in running_pids: - # verify the correct signal on the mock - self.assertEquals(swift_init.os.pid_sigs[pid], - [signal.SIG_DFL]) - current_pids = list(PID_MAP[server.server]) - # each pass will assert the pid from the previous run was removed - last_pid = -1 # this pid is skipped on the first pass - for pid in list(current_pids): - # setup current running pids - swift_init.os = MockOs(current_pids) - if not (last_pid == -1): - # any valid pid should show up in iter_pid_files - self.assert_(last_pid in [ - p for f, p in list(server.iter_pid_files())]) - running_pids = server.get_running_pids() - self.assert_(pid in running_pids) - self.assertFalse(last_pid in running_pids) - # assert stale pid files were removed! - self.assertEquals(len(running_pids), len(server.pid_files())) - if last_pid in ALL_PIDS: - self.assertFalse(last_pid in [ - p for f, p in list(server.iter_pid_files())]) - # get ready for next iter, simulate pid died (i.e. remove # pid) - last_pid = pid - current_pids.remove(pid) - self.assertEquals(len(current_pids), 0) - swift_init.os = MockOs(current_pids) - self.assertEquals(len(server.get_running_pids()), 0) - - with temptree(*zip(*PID_FILES)) as t: + # test only gets running pids + pid_files = ( + ('test-server1.pid', 1), + ('test-server2.pid', 2), + ) + with temptree(*zip(*pid_files)) as t: swift_init.RUN_DIR = t - servers = \ - 'auth proxy object object-auditor container-replicator'.split() - for server in servers: - assert_running_pids(server) + server = swift_init.SwiftServer('test-server') + # mock os, only pid '1' is running + swift_init.os = MockOs([1]) + running_pids = server.get_running_pids() + self.assertEquals(len(running_pids), 1) + self.assert_(1 in running_pids) + self.assert_(2 not in running_pids) + # test persistant running pid files + self.assert_(os.path.exists(os.path.join(t, 'test-server1.pid'))) + # test clean up stale pids + self.assertFalse(os.path.exists(os.path.join(t, 'test-server2.pid'))) + # reset mock os, no pids running + swift_init.os = MockOs([]) + running_pids = server.get_running_pids() + self.assertFalse(running_pids) + # and now all pid files are cleaned out + self.assertFalse(os.path.exists(os.path.join(t, 'test-server1.pid'))) + all_pids = os.listdir(t) + self.assertEquals(len(all_pids), 0) + + # test only get pids for right server + pid_files = ( + ('thing-doer.pid', 1), + ('thing-sayer.pid', 2), + ('other-doer.pid', 3), + ('other-sayer.pid', 4), + ) + files, pids = zip(*pid_files) + with temptree(files, pids) as t: + swift_init.RUN_DIR = t + # all pids are running + swift_init.os = MockOs(pids) + server = swift_init.SwiftServer('thing-doer') + running_pids = server.get_running_pids() + # only thing-doer.pid, 1 + self.assertEquals(len(running_pids), 1) + self.assert_(1 in running_pids) + # no other pids returned + for n in (2, 3, 4): + self.assert_(n not in running_pids) + # assert stale pids for other servers ignored + swift_init.os = MockOs([1]) # only thing-doer is running + running_pids = server.get_running_pids() + for f in ('thing-sayer.pid', 'other-doer.pid', 'other-sayer.pid'): + # other server pid files persist + self.assert_(os.path.exists, os.path.join(t, f)) + # verify that servers are in fact not running + for server_name in ('thing-sayer', 'other-doer', 'other-sayer'): + server = swift_init.SwiftServer(server_name) + running_pids = server.get_running_pids() + self.assertFalse(running_pids) + # and now all OTHER pid files are cleaned out + all_pids = os.listdir(t) + self.assertEquals(len(all_pids), 1) + self.assert_(os.path.exists(os.path.join(t, 'thing-doer.pid'))) + def test_kill_running_pids(self): - def assert_kill_running_pids(server): - server = swift_init.SwiftServer(server) - # all pids running - swift_init.os = MockOs(ALL_PIDS) - - pid_files = server.pid_files() - killed_pids = server.kill_running_pids() - self.assertEquals(len(killed_pids), len(pid_files)) - - # reinit mock os - swift_init.os = MockOs(ALL_PIDS) - # make sure graceful sends sighup on servers that support it - huped_pids = server.kill_running_pids(graceful=True) - if server.server in swift_init.GRACEFUL_SHUTDOWN_SERVERS: - captured_sig = signal.SIGHUP - else: - captured_sig = signal.SIGTERM - for pid in huped_pids: - self.assertEquals(swift_init.os.pid_sigs[pid], [captured_sig]) - - with temptree(*zip(*PID_FILES)) as t: + pid_files = ( + ('object-server.pid', 1), + ('object-replicator1.pid', 11), + ('object-replicator2.pid', 12), + ) + files, pids = zip(*pid_files) + with temptree(files, pids) as t: swift_init.RUN_DIR = t - servers = \ - 'auth proxy object object-auditor container-replicator'.split() - for server in servers: - assert_kill_running_pids(server) + server = swift_init.SwiftServer('object') + # test no servers running + pids = server.kill_running_pids() + self.assertFalse(pids) + # start up pid + swift_init.os = MockOs([1]) + # test kill one pid + pids = server.kill_running_pids() + self.assertEquals(len(pids), 1) + self.assert_(1 in pids) + self.assertEquals(swift_init.os.pid_sigs[1], [signal.SIGTERM]) + # reset os mock + swift_init.os = MockOs([1]) + # test shutdown + self.assert_('object-server' in swift_init.GRACEFUL_SHUTDOWN_SERVERS) + pids = server.kill_running_pids(graceful=True) + self.assertEquals(len(pids), 1) + self.assert_(1 in pids) + self.assertEquals(swift_init.os.pid_sigs[1], [signal.SIGHUP]) + # start up other servers + swift_init.os = MockOs([11, 12]) + # test multi server kill & ignore graceful on unsupport server + self.assertFalse('object-replicator' in + swift_init.GRACEFUL_SHUTDOWN_SERVERS) + server = swift_init.SwiftServer('object-replicator') + pids = server.kill_running_pids(graceful=True) + self.assertEquals(len(pids), 2) + for pid in (11, 12): + self.assert_(pid in pids) + self.assertEquals(swift_init.os.pid_sigs[pid], + [signal.SIGTERM]) + # and the other pid is of course not signaled + self.assert_(1 not in swift_init.os.pid_sigs) + #TODO: more tests def test_status(self): From 6c37cc44df58d68f96af8b360b2bfee0d115401b Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Fri, 19 Nov 2010 07:42:21 -0600 Subject: [PATCH 03/64] started work on new tests --- test/bin/test_swift_init.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/test/bin/test_swift_init.py b/test/bin/test_swift_init.py index 37082c4eb2..1ed159c13a 100644 --- a/test/bin/test_swift_init.py +++ b/test/bin/test_swift_init.py @@ -566,11 +566,24 @@ class TestSwiftServerClass(unittest.TestCase): # and the other pid is of course not signaled self.assert_(1 not in swift_init.os.pid_sigs) - - #TODO: more tests def test_status(self): - pass + # test supplied pids doesn't call get_running_pids + called = [] + def mock(*args, **kwargs): + called.append(True) + server = swift_init.SwiftServer('test') + server.get_running_pids = mock + pids = { + 1: 'test-server.pid' + } + server.status(pids=pids) + self.assertFalse(called) + # and again with out pids + server.status() + self.assert_(called) + + #TODO: more tests def test_spawn(self): pass From b034e6001a1ce80486245a62603570a8efe64a0c Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 24 Nov 2010 15:22:45 -0600 Subject: [PATCH 04/64] new test for status; pep8; moved capture std.out to inside temptree --- test/bin/test_swift_init.py | 108 +++++++++++++++++++++++++++++++----- 1 file changed, 93 insertions(+), 15 deletions(-) diff --git a/test/bin/test_swift_init.py b/test/bin/test_swift_init.py index 37082c4eb2..e9a136e37f 100644 --- a/test/bin/test_swift_init.py +++ b/test/bin/test_swift_init.py @@ -28,6 +28,7 @@ from test.bin import swift_init # for testing... DUMMY_SIG = 1 + @contextmanager def temptree(files, contents=''): # generate enough contents to fill the files @@ -79,9 +80,10 @@ def pop_stream(f): output = f.read() f.seek(0) f.truncate() - print >> sys.stderr, output + #print >> sys.stderr, output return output + class TestSwiftInitModule(unittest.TestCase): def test_servers(self): @@ -246,7 +248,8 @@ class TestSwiftServerClass(unittest.TestCase): ini_files = server.ini_files() self.assertEquals(len(ini_files), 1) ini_file = ini_files[0] - self.assertEquals(ini_file, self.join_swift_dir('proxy-server.conf')) + proxy_conf = self.join_swift_dir('proxy-server.conf') + self.assertEquals(ini_file, proxy_conf) # test multi server conf files & grouping of server-type config ini_files = ( @@ -272,7 +275,6 @@ class TestSwiftServerClass(unittest.TestCase): sorted_confs = sorted([c1, c2, c3, c4]) self.assertEquals(ini_files, sorted_confs) - # test get single numbered conf ini_files = ( 'account-server/1.conf', @@ -301,7 +303,7 @@ class TestSwiftServerClass(unittest.TestCase): swift_init.SWIFT_DIR = t old_stdout = sys.stdout try: - with open('std.out', 'w+') as f: + with open(os.path.join(t, 'output'), 'w+') as f: sys.stdout = f server = swift_init.SwiftServer('auth') # check warn "unable to locate" @@ -399,7 +401,8 @@ class TestSwiftServerClass(unittest.TestCase): self.assertEquals(len(pids), 1) pid_file, pid = pids[0] self.assertEquals(pid, 2) - self.assertEquals(pid_file, self.join_run_dir('object-server/2.pid')) + pid_two = self.join_run_dir('object-server/2.pid') + self.assertEquals(pid_file, pid_two) # try to iter on a pid number with a matching conf but no pid pids = list(server.iter_pid_files(number=3)) self.assertFalse(pids) @@ -428,7 +431,7 @@ class TestSwiftServerClass(unittest.TestCase): # capture stdio old_stdout = sys.stdout try: - with open('std.out', 'w+') as f: + with open(os.path.join(t, 'output'), 'w+') as f: sys.stdout = f #test print details pids = server.signal_pids(DUMMY_SIG) @@ -454,7 +457,8 @@ class TestSwiftServerClass(unittest.TestCase): pids = server.signal_pids(signal.SIG_DFL, verbose=True) output = pop_stream(f) self.assert_('stale pid' in output.lower()) - self.assert_(self.join_run_dir('auth-server.pid') in output) + auth_pid = self.join_run_dir('auth-server.pid') + self.assert_(auth_pid in output) finally: sys.stdout = old_stdout @@ -476,13 +480,15 @@ class TestSwiftServerClass(unittest.TestCase): # test persistant running pid files self.assert_(os.path.exists(os.path.join(t, 'test-server1.pid'))) # test clean up stale pids - self.assertFalse(os.path.exists(os.path.join(t, 'test-server2.pid'))) + pid_two = self.join_swift_dir('test-server2.pid') + self.assertFalse(os.path.exists(pid_two)) # reset mock os, no pids running swift_init.os = MockOs([]) running_pids = server.get_running_pids() self.assertFalse(running_pids) # and now all pid files are cleaned out - self.assertFalse(os.path.exists(os.path.join(t, 'test-server1.pid'))) + pid_one = self.join_run_dir('test-server1.pid') + self.assertFalse(os.path.exists(pid_one)) all_pids = os.listdir(t) self.assertEquals(len(all_pids), 0) @@ -500,7 +506,7 @@ class TestSwiftServerClass(unittest.TestCase): swift_init.os = MockOs(pids) server = swift_init.SwiftServer('thing-doer') running_pids = server.get_running_pids() - # only thing-doer.pid, 1 + # only thing-doer.pid, 1 self.assertEquals(len(running_pids), 1) self.assert_(1 in running_pids) # no other pids returned @@ -522,7 +528,6 @@ class TestSwiftServerClass(unittest.TestCase): self.assertEquals(len(all_pids), 1) self.assert_(os.path.exists(os.path.join(t, 'thing-doer.pid'))) - def test_kill_running_pids(self): pid_files = ( ('object-server.pid', 1), @@ -546,7 +551,8 @@ class TestSwiftServerClass(unittest.TestCase): # reset os mock swift_init.os = MockOs([1]) # test shutdown - self.assert_('object-server' in swift_init.GRACEFUL_SHUTDOWN_SERVERS) + self.assert_('object-server' in + swift_init.GRACEFUL_SHUTDOWN_SERVERS) pids = server.kill_running_pids(graceful=True) self.assertEquals(len(pids), 1) self.assert_(1 in pids) @@ -566,11 +572,83 @@ class TestSwiftServerClass(unittest.TestCase): # and the other pid is of course not signaled self.assert_(1 not in swift_init.os.pid_sigs) + def test_status(self): + ini_files = ( + 'test-server/1.conf', + 'test-server/2.conf', + 'test-server/3.conf', + 'test-server/4.conf', + ) + + pid_files = ( + ('test-server/1.pid', 1), + ('test-server/2.pid', 2), + ('test-server/3.pid', 3), + ('test-server/4.pid', 4), + ) + + with temptree(ini_files) as swift_dir: + swift_init.SWIFT_DIR = swift_dir + files, pids = zip(*pid_files) + with temptree(files, pids) as t: + swift_init.RUN_DIR = t + # setup running servers + server = swift_init.SwiftServer('test') + # capture stdio + old_stdout = sys.stdout + try: + with open(os.path.join(t, 'output'), 'w+') as f: + sys.stdout = f + # test status for all running + swift_init.os = MockOs(pids) + self.assertEquals(server.status(), 0) + output = pop_stream(f).strip().splitlines() + self.assertEquals(len(output), 4) + for line in output: + self.assert_('test-server running' in line) + # test get single server by number + self.assertEquals(server.status(number=4), 0) + output = pop_stream(f).strip().splitlines() + self.assertEquals(len(output), 1) + line = output[0] + self.assert_('test-server running' in line) + conf_four = self.join_swift_dir(ini_files[3]) + self.assert_('4 - %s' % conf_four in line) + # test some servers not running + swift_init.os = MockOs([1, 2, 3]) + self.assertEquals(server.status(), 0) + output = pop_stream(f).strip().splitlines() + self.assertEquals(len(output), 3) + for line in output: + self.assert_('test-server running' in line) + # test single server not running + swift_init.os = MockOs([1, 2]) + self.assertEquals(server.status(number=3), 1) + output = pop_stream(f).strip().splitlines() + self.assertEquals(len(output), 1) + line = output[0] + self.assert_('not running' in line) + conf_three = self.join_swift_dir(ini_files[2]) + self.assert_(conf_three in line) + # test no running pids + swift_init.os = MockOs([]) + self.assertEquals(server.status(), 1) + output = pop_stream(f).lower() + self.assert_('no test-server running' in output) + # test use provided pids + pids = { + 1: '1.pid', + 2: '2.pid', + } + self.assertEquals(server.status(pids=pids), 0) + output = pop_stream(f).strip().splitlines() + self.assertEquals(len(output), 2) + for line in output: + self.assert_('test-server running' in line) + finally: + sys.stdout = old_stdout #TODO: more tests - def test_status(self): - pass - def test_spawn(self): pass From cf70f54e870b9c34b1c075a704fca5e3f108ebf8 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 7 Dec 2010 17:30:04 -0600 Subject: [PATCH 05/64] finished test_spawn, started test_wait --- bin/swift-init | 3 +- test/bin/test_swift_init.py | 192 +++++++++++++++++++++++++++++++++++- 2 files changed, 190 insertions(+), 5 deletions(-) diff --git a/bin/swift-init b/bin/swift-init index b537ba8341..ea55f6a65c 100755 --- a/bin/swift-init +++ b/bin/swift-init @@ -100,8 +100,7 @@ def write_file(path, contents): os.makedirs(dir) except OSError, err: if err.errno == errno.EACCES: - sys.exit('Unable to create %s. Running as non-root?' % - dir) + sys.exit('Unable to create %s. Running as non-root?' % dir) with open(path, 'w') as f: f.write('%s' % contents) diff --git a/test/bin/test_swift_init.py b/test/bin/test_swift_init.py index 82437aff6f..fcb5dd31d0 100644 --- a/test/bin/test_swift_init.py +++ b/test/bin/test_swift_init.py @@ -22,6 +22,8 @@ from shutil import rmtree from contextlib import contextmanager from tempfile import mkdtemp from collections import defaultdict +from threading import Thread +from time import sleep from test import bin # for reloading... from test.bin import swift_init # for testing... @@ -93,6 +95,10 @@ class TestSwiftInitModule(unittest.TestCase): self.assertEquals(len(main_plus_rest), len(swift_init.MAIN_SERVERS) + len(swift_init.REST_SERVERS)) + def test_setup_env(self): + # TODO: tests + pass + def test_search_tree(self): # file match & ext miss with temptree(['asdf.conf', 'blarg.conf', 'asdf.cfg']) as t: @@ -150,6 +156,16 @@ class TestSwiftInitModule(unittest.TestCase): with open(file_name, 'r') as f: contents = f.read() self.assertEquals(contents, 'test') + # and also subdirs + file_name = os.path.join(t, 'subdir/test2') + swift_init.write_file(file_name, 'test2') + with open(file_name, 'r') as f: + contents = f.read() + self.assertEquals(contents, 'test2') + # but can't over-write files + file_name = os.path.join(t, 'subdir/test2/test3') + self.assertRaises(IOError, swift_init.write_file, file_name, + 'test3') def test_remove_file(self): with temptree([]) as t: @@ -656,12 +672,182 @@ class TestSwiftServerClass(unittest.TestCase): finally: sys.stdout = old_stdout - #TODO: more tests def test_spawn(self): - pass + # mocks + class MockProcess(): + + NOTHING = 'default besides None' + STDOUT = 'stdout' + PIPE = 'pipe' + + def __init__(self, pids=None): + if pids is None: + pids = [] + self.pids = (p for p in pids) + + def Popen(self, args, **kwargs): + return MockProc(self.pids.next(), args, **kwargs) + + class MockProc(): + + def __init__(self, pid, args, stdout=MockProcess.NOTHING, + stderr=MockProcess.NOTHING): + self.pid = pid + self.args = args + self.stdout = stdout + if stderr == MockProcess.STDOUT: + self.stderr = self.stdout + else: + self.stderr = stderr + + # setup running servers + server = swift_init.SwiftServer('test') + + with temptree(['test-server.conf']) as swift_dir: + swift_init.SWIFT_DIR = swift_dir + with temptree([]) as t: + swift_init.RUN_DIR = t + old_subprocess = swift_init.subprocess + try: + # test single server process calls spawn once + swift_init.subprocess = MockProcess([1]) + conf_file = self.join_swift_dir('test-server.conf') + # spawn server no kwargs + server.spawn(conf_file) + # test pid file + pid_file = self.join_run_dir('test-server.pid') + self.assert_(os.path.exists(pid_file)) + pid_on_disk = int(open(pid_file).read().strip()) + self.assertEquals(pid_on_disk, 1) + # assert procs args + self.assert_(server.procs) + self.assertEquals(len(server.procs), 1) + proc = server.procs[0] + expected_args = [ + 'swift-test-server', + conf_file, + ] + self.assertEquals(proc.args, expected_args) + # assert stdout is /dev/null + self.assert_(isinstance(proc.stdout, file)) + self.assertEquals(proc.stdout.name, os.devnull) + self.assertEquals(proc.stdout.mode, 'w+b') + self.assertEquals(proc.stderr, proc.stdout) + # test multi server process calls spawn multiple times + swift_init.subprocess = MockProcess([11, 12, 13, 14]) + conf1 = self.join_swift_dir('test-server/1.conf') + conf2 = self.join_swift_dir('test-server/2.conf') + conf3 = self.join_swift_dir('test-server/3.conf') + conf4 = self.join_swift_dir('test-server/4.conf') + server = swift_init.SwiftServer('test') + # test server run once + server.spawn(conf1, once=True) + self.assert_(server.procs) + self.assertEquals(len(server.procs), 1) + proc = server.procs[0] + expected_args = ['swift-test-server', conf1, 'once'] + self.assertEquals(proc.args, expected_args) + # assert stdout is /dev/null + self.assert_(isinstance(proc.stdout, file)) + self.assertEquals(proc.stdout.name, os.devnull) + self.assertEquals(proc.stdout.mode, 'w+b') + self.assertEquals(proc.stderr, proc.stdout) + # test server not daemon + server.spawn(conf2, daemon=False) + self.assert_(server.procs) + self.assertEquals(len(server.procs), 2) + proc = server.procs[1] + expected_args = ['swift-test-server', conf2, 'verbose'] + self.assertEquals(proc.args, expected_args) + # assert stdout is not changed + self.assertEquals(proc.stdout, None) + self.assertEquals(proc.stderr, None) + # test server wait + server.spawn(conf3, wait=True) + self.assert_(server.procs) + self.assertEquals(len(server.procs), 3) + proc = server.procs[2] + # assert stdout is piped + self.assertEquals(proc.stdout, MockProcess.PIPE) + self.assertEquals(proc.stderr, proc.stdout) + # test not daemon over-rides wait + server.spawn(conf4, wait=True, daemon=False, once=True) + self.assert_(server.procs) + self.assertEquals(len(server.procs), 4) + proc = server.procs[3] + expected_args = ['swift-test-server', conf4, 'once', + 'verbose'] + self.assertEquals(proc.args, expected_args) + # daemon behavior should trump wait, once shouldn't matter + self.assertEquals(proc.stdout, None) + self.assertEquals(proc.stderr, None) + # assert pids + for i, proc in enumerate(server.procs): + pid_file = self.join_run_dir('test-server/%d.pid' % + (i + 1)) + pid_on_disk = int(open(pid_file).read().strip()) + self.assertEquals(pid_on_disk, proc.pid) + finally: + swift_init.subprocess = old_subprocess + + #TODO: more tests def test_wait(self): - pass + server = swift_init.SwiftServer('test') + self.assertEquals(server.wait(), 0) + + class MockProcess(Thread): + def __init__(self, stdout, delay=0.1, fail_to_start=False): + Thread.__init__(self) + self.stdout = stdout + self.delay = delay + self.finished = False + self.returncode = None + if fail_to_start: + self.run = self.fail + + def close_stdout(self): + with open(os.devnull, 'r+b') as nullfile: + try: + os.dup2(nullfile.fileno(), self.stdout.fileno()) + except OSError: + pass + + def fail(self): + print >>self.stdout, 'mock process started' + sleep(self.delay) # perform setup processing + print >>self.stdout, 'mock process failed to start' + self.returncode = 1 + self.close_stdout() + self.finished = True + + def run(self): + print >>self.stdout, 'mock process started' + sleep(self.delay) # perform setup processing + print >>self.stdout, 'setup complete!' + self.close_stdout() + sleep(self.delay) # do some more processing + print >>self.stdout, 'mock process finished' + self.returncode = 0 + self.finished = True + + with temptree([]) as t: + old_stdout = sys.stdout + try: + with open(os.path.join(t, 'output'), 'w+') as f: + # acctually capture the read stdout (for prints) + sys.stdout = f + stdout = open(os.path.join(t, 'subout'), 'w+') + # the proc will have it's stdout redirected to + # subprocess.PIPE + proc = MockProcess(stdout) + proc.start() + server.procs = [proc] + server.wait(output=True) + proc.join() + print >>sys.stderr, pop_stream(f) + finally: + sys.stdout = old_stdout def test_interact(self): pass From c10251a384e8c5d003eb737b12e75e1afb37dab5 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 8 Dec 2010 01:12:21 -0600 Subject: [PATCH 06/64] finished tests for wait --- test/bin/test_swift_init.py | 80 +++++++++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 21 deletions(-) diff --git a/test/bin/test_swift_init.py b/test/bin/test_swift_init.py index fcb5dd31d0..532774ac1d 100644 --- a/test/bin/test_swift_init.py +++ b/test/bin/test_swift_init.py @@ -791,44 +791,55 @@ class TestSwiftServerClass(unittest.TestCase): finally: swift_init.subprocess = old_subprocess - #TODO: more tests def test_wait(self): server = swift_init.SwiftServer('test') self.assertEquals(server.wait(), 0) class MockProcess(Thread): - def __init__(self, stdout, delay=0.1, fail_to_start=False): + def __init__(self, delay=0.1, fail_to_start=False): Thread.__init__(self) - self.stdout = stdout + # setup pipe + rfd, wfd = os.pipe() + # subprocess connection to read stdout + self.stdout = os.fdopen(rfd) + # real process connection to write stdout + self._stdout = os.fdopen(wfd, 'w') self.delay = delay self.finished = False self.returncode = None if fail_to_start: self.run = self.fail + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + if self.isAlive(): + self.join() + def close_stdout(self): - with open(os.devnull, 'r+b') as nullfile: + self._stdout.flush() + with open(os.devnull, 'wb') as nullfile: try: - os.dup2(nullfile.fileno(), self.stdout.fileno()) + os.dup2(nullfile.fileno(), self._stdout.fileno()) except OSError: pass def fail(self): - print >>self.stdout, 'mock process started' + print >>self._stdout, 'mock process started' sleep(self.delay) # perform setup processing - print >>self.stdout, 'mock process failed to start' + print >>self._stdout, 'mock process failed to start' self.returncode = 1 self.close_stdout() - self.finished = True def run(self): - print >>self.stdout, 'mock process started' + print >>self._stdout, 'mock process started' sleep(self.delay) # perform setup processing - print >>self.stdout, 'setup complete!' + print >>self._stdout, 'setup complete!' self.close_stdout() sleep(self.delay) # do some more processing - print >>self.stdout, 'mock process finished' - self.returncode = 0 + print >>self._stdout, 'mock process finished' self.finished = True with temptree([]) as t: @@ -837,18 +848,45 @@ class TestSwiftServerClass(unittest.TestCase): with open(os.path.join(t, 'output'), 'w+') as f: # acctually capture the read stdout (for prints) sys.stdout = f - stdout = open(os.path.join(t, 'subout'), 'w+') - # the proc will have it's stdout redirected to - # subprocess.PIPE - proc = MockProcess(stdout) - proc.start() - server.procs = [proc] - server.wait(output=True) - proc.join() - print >>sys.stderr, pop_stream(f) + # test closing pipe in subprocess unblocks read + with MockProcess() as proc: + server.procs = [proc] + status = server.wait() + self.assertEquals(status, 0) + # wait should return as soon as stdout is closed + self.assert_(proc.isAlive()) + self.assertFalse(proc.finished) + self.assert_(proc.finished) # make sure it did finish... + # test output kwarg prints subprocess output + with MockProcess() as proc: + server.procs = [proc] + status = server.wait(output=True) + output = pop_stream(f) + self.assert_('mock process started' in output) + self.assert_('setup complete' in output) + # make sure we don't get prints after stdout was closed + self.assertFalse('mock process finished' in output) + # test process which fails to start + with MockProcess(fail_to_start=True) as proc: + server.procs = [proc] + status = server.wait() + self.assertEquals(status, 1) + self.assert_('failed' in pop_stream(f)) + # test multiple procs + procs = [MockProcess() for i in range(3)] + for proc in procs: + proc.start() + server.procs = procs + status = server.wait() + self.assertEquals(status, 0) + for proc in procs: + self.assert_(proc.isAlive()) + for proc in procs: + proc.join() finally: sys.stdout = old_stdout + #TODO: more tests def test_interact(self): pass From 668666c18b52372e3f6c49661b62a474a94eb4d7 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 9 Dec 2010 01:42:49 -0600 Subject: [PATCH 07/64] mostly finished SwiftServer tests, needs some cleanup --- bin/swift-init | 8 +- test/bin/test_swift_init.py | 195 +++++++++++++++++++++++++++++++++++- 2 files changed, 197 insertions(+), 6 deletions(-) diff --git a/bin/swift-init b/bin/swift-init index ea55f6a65c..b38b423c74 100755 --- a/bin/swift-init +++ b/bin/swift-init @@ -535,6 +535,8 @@ class SwiftServer(): :param once: boolean, add once argument to command :param wait: boolean, if true capture stdout with a pipe :param daemon: boolean, if true ask server to log to console + + :returns : the pid of the spawned process """ args = [self.cmd, ini_file] if once: @@ -559,6 +561,7 @@ class SwiftServer(): pid_file = self.get_pid_file_name(ini_file) write_file(pid_file, proc.pid) self.procs.append(proc) + return proc.pid def wait(self, **kwargs): """ @@ -603,9 +606,12 @@ class SwiftServer(): # already running (unless -n specifies which one you want), this # restriction could potentially be lifted, and launch could start # any unstarted instances - if ini_file in ini_files or not number: + if ini_file in ini_files: already_started = True print "%s running (%s - %s)" % (self.server, pid, ini_file) + elif not kwargs.get('number', 0): + already_started = True + print "%s running (%s - %s)" % (self.server, pid, pid_file) if already_started: print "%s already started..." % self.server diff --git a/test/bin/test_swift_init.py b/test/bin/test_swift_init.py index 532774ac1d..3d4fed2f5a 100644 --- a/test/bin/test_swift_init.py +++ b/test/bin/test_swift_init.py @@ -865,7 +865,7 @@ class TestSwiftServerClass(unittest.TestCase): self.assert_('mock process started' in output) self.assert_('setup complete' in output) # make sure we don't get prints after stdout was closed - self.assertFalse('mock process finished' in output) + self.assert_('mock process finished' not in output) # test process which fails to start with MockProcess(fail_to_start=True) as proc: server.procs = [proc] @@ -886,16 +886,201 @@ class TestSwiftServerClass(unittest.TestCase): finally: sys.stdout = old_stdout - #TODO: more tests def test_interact(self): - pass + class MockProcess(): + def __init__(self, fail=False): + self.returncode = None + if fail: + self._returncode = 1 + else: + self._returncode = 0 + + def communicate(self): + self.returncode = self._returncode + return '', '' + + server = swift_init.SwiftServer('test') + server.procs = [MockProcess()] + self.assertEquals(server.interact(), 0) + server.procs = [MockProcess(fail=True)] + self.assertEquals(server.interact(), 1) + procs = [] + for fail in (False, True, True): + procs.append(MockProcess(fail=fail)) + server.procs = procs + self.assert_(server.interact() > 0) + def test_launch(self): - pass + # stubs + ini_files = ( + 'proxy-server.conf', + 'object-server/1.conf', + 'object-server/2.conf', + 'object-server/3.conf', + 'object-server/4.conf', + ) + pid_files = ( + ('proxy-server.pid', 1), + ('proxy-server/2.pid', 2), + ) + #mocks + class MockSpawn(): + + def __init__(self, pids=None): + self.ini_files = [] + self.kwargs = [] + if not pids: + def one_forever(): + while True: + yield 1 + self.pids = one_forever() + else: + self.pids = (x for x in pids) + + def __call__(self, ini_file, **kwargs): + self.ini_files.append(ini_file) + self.kwargs.append(kwargs) + return self.pids.next() + + with temptree(ini_files) as swift_dir: + swift_init.SWIFT_DIR = swift_dir + files, pids = zip(*pid_files) + with temptree(files, pids) as t: + swift_init.RUN_DIR = t + old_stdout = sys.stdout + try: + with open(os.path.join(t, 'output'), 'w+') as f: + sys.stdout = f + # can't start server w/o an conf + server = swift_init.SwiftServer('test') + self.assertFalse(server.launch()) + # start mock os running all pids + swift_init.os = MockOs(pids) + server = swift_init.SwiftServer('proxy') + # can't start server if it's already running + self.assertFalse(server.launch()) + output = pop_stream(f) + self.assert_('running' in output) + ini_file = self.join_swift_dir('proxy-server.conf') + self.assert_(ini_file in output) + pid_file = self.join_run_dir('proxy-server/2.pid') + self.assert_(pid_file in output) + self.assert_('already started' in output) + # no running pids + swift_init.os = MockOs([]) + # test ignore once for non-start-once server + mock_spawn = MockSpawn([1]) + server.spawn = mock_spawn + ini_file = self.join_swift_dir('proxy-server.conf') + expected = { + 1: ini_file, + } + self.assertEquals(server.launch(once=True), expected) + self.assertEquals(mock_spawn.ini_files, [ini_file]) + expected = { + 'once': False, + } + self.assertEquals(mock_spawn.kwargs, [expected]) + output = pop_stream(f) + self.assert_('Starting' in output) + self.assert_('once' not in output) + # test multi-server kwarg once + server = swift_init.SwiftServer('object-replicator') + mock_spawn = MockSpawn([1, 2, 3, 4]) + server.spawn = mock_spawn + conf1 = self.join_swift_dir('object-server/1.conf') + conf2 = self.join_swift_dir('object-server/2.conf') + conf3 = self.join_swift_dir('object-server/3.conf') + conf4 = self.join_swift_dir('object-server/4.conf') + expected = { + 1: conf1, + 2: conf2, + 3: conf3, + 4: conf4, + } + self.assertEquals(server.launch(once=True), expected) + self.assertEquals(mock_spawn.ini_files, [conf1, conf2, + conf3, conf4]) + expected = { + 'once': True, + } + self.assertEquals(len(mock_spawn.kwargs), 4) + for kwargs in mock_spawn.kwargs: + self.assertEquals(kwargs, expected) + # test number kwarg + mock_spawn = MockSpawn([4]) + server.spawn = mock_spawn + expected = { + 4: conf4, + } + self.assertEquals(server.launch(number=4), expected) + self.assertEquals(mock_spawn.ini_files, [conf4]) + expected = { + 'number': 4 + } + self.assertEquals(mock_spawn.kwargs, [expected]) + + + finally: + sys.stdout = old_stdout + + #TODO: more tests def test_stop(self): - pass + ini_files = ( + 'account-server/1.conf', + 'account-server/2.conf', + 'account-server/3.conf', + 'account-server/4.conf', + ) + pid_files = ( + ('account-reaper/1.pid', 1), + ('account-reaper/2.pid', 2), + ('account-reaper/3.pid', 3), + ('account-reaper/4.pid', 4), + ) + with temptree(ini_files) as swift_dir: + swift_init.SWIFT_DIR = swift_dir + files, pids = zip(*pid_files) + with temptree(files, pids) as t: + swift_init.RUN_DIR = t + # start all pids in mock os + swift_init.os = MockOs(pids) + server = swift_init.SwiftServer('account-reaper') + # test kill all running pids + pids = server.stop() + self.assertEquals(len(pids), 4) + for pid in (1, 2, 3, 4): + self.assert_(pid in pids) + self.assertEquals(swift_init.os.pid_sigs[pid], + [signal.SIGTERM]) + conf1 = self.join_swift_dir('account-reaper/1.conf') + conf2 = self.join_swift_dir('account-reaper/2.conf') + conf3 = self.join_swift_dir('account-reaper/3.conf') + conf4 = self.join_swift_dir('account-reaper/4.conf') + # reset mock os with only 2 running pids + swift_init.os = MockOs([3, 4]) + pids = server.stop() + self.assertEquals(len(pids), 2) + for pid in (3, 4): + self.assert_(pid in pids) + self.assertEquals(swift_init.os.pid_sigs[pid], + [signal.SIGTERM]) + self.assertFalse(os.path.exists(conf1)) + self.assertFalse(os.path.exists(conf2)) + # test number kwarg + swift_init.os = MockOs([3, 4]) + pids = server.stop(number=3) + self.assertEquals(len(pids), 1) + expected = { + 3: conf3, + } + self.assert_(pids, expected) + self.assertEquals(swift_init.os.pid_sigs[3], [signal.SIGTERM]) + self.assertFalse(os.path.exists(conf4)) + self.assertFalse(os.path.exists(conf3)) #TODO: test SwiftInit class class TestSwiftInitClass(unittest.TestCase): From ef487c65ef1ed6b4056459c563bdc78f9d379a98 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 29 Dec 2010 00:01:30 -0600 Subject: [PATCH 08/64] good start on TestSwiftInitClass --- bin/swift-init | 42 +++++--- test/bin/test_swift_init.py | 207 ++++++++++++++++++++++++++++++++++-- 2 files changed, 225 insertions(+), 24 deletions(-) diff --git a/bin/swift-init b/bin/swift-init index b38b423c74..cd56e412be 100755 --- a/bin/swift-init +++ b/bin/swift-init @@ -144,8 +144,6 @@ class SwiftInit(): """ def __init__(self, servers): - self.servers = [] - server_names = set() for server in servers: if server == 'all': @@ -157,8 +155,9 @@ class SwiftInit(): else: server_names.add(server) + self.servers = set() for name in server_names: - self.servers.append(SwiftServer(name)) + self.servers.add(SwiftServer(name)) def watch_server_pids(self, server_pids, interval=0, **kwargs): """Monitor a collection of server pids yeilding back those pids that @@ -331,20 +330,17 @@ class SwiftInit(): raise UnknownCommand(cmd) return f - def list_commands(self): + @classmethod + def list_commands(cls): """Get all publicly accessible commands :returns: a list of strings, the method names who are decorated as commands """ - cmds = sorted([x.replace('_', '-') for x in dir(self) if \ - hasattr(getattr(self, x), 'publicly_accessible')]) - try: - helps = [self.get_command(x).__doc__.strip() for x in cmds] - except AttributeError: - raise AttributeError( - 'command %s has no __doc__, please add one' % x) - return zip(cmds, helps) + get_method = lambda cmd: getattr(cls, cmd) + return sorted([(x.replace('_', '-'), get_method(x).__doc__.strip()) + for x in dir(cls) if + getattr(get_method(x), 'publicly_accessible', False)]) def run_command(self, cmd, **kwargs): """Find the named command and run it @@ -365,7 +361,7 @@ class SwiftServer(): def __init__(self, server): if '-' not in server: server = '%s-server' % server - self.server = server + self.server = server.lower() self.type = '-'.join(server.split('-')[:-1]) self.cmd = 'swift-%s' % server self.procs = [] @@ -373,6 +369,18 @@ class SwiftServer(): def __str__(self): return self.server + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, repr(str(self))) + + def __hash__(self): + return hash(str(self)) + + def __eq__(self, other): + try: + return self.server == other.server + except AttributeError: + return False + def get_pid_file_name(self, ini_file): """Translate ini_file to a corresponding pid_file @@ -645,7 +653,7 @@ class SwiftServer(): USAGE = """%prog [ Commands: -""" + '\n'.join(["%16s: %s" % x for x in SwiftInit([]).list_commands()]) +""" + '\n'.join(["%16s: %s" % x for x in SwiftInit.list_commands()]) def main(): @@ -676,13 +684,13 @@ def main(): if len(servers) == 1: # this is just a stupid swap for me cause I always try to "start main" - commands, docs = zip(*SwiftInit([]).list_commands()) + commands, docs = zip(*SwiftInit.list_commands()) if servers[0] in commands: command, servers = servers[0], [command] - init = SwiftInit(servers) + controller = SwiftInit(servers) try: - status = init.run_command(command, **options.__dict__) + status = controller.run_command(command, **options.__dict__) except UnknownCommand: parser.print_help() print 'ERROR: unknown command, %s' % command diff --git a/test/bin/test_swift_init.py b/test/bin/test_swift_init.py index 3d4fed2f5a..0d141f15b2 100644 --- a/test/bin/test_swift_init.py +++ b/test/bin/test_swift_init.py @@ -14,6 +14,7 @@ # limitations under the License. import unittest +from nose import SkipTest import os import sys @@ -97,7 +98,7 @@ class TestSwiftInitModule(unittest.TestCase): def test_setup_env(self): # TODO: tests - pass + raise SkipTest def test_search_tree(self): # file match & ext miss @@ -209,7 +210,7 @@ class TestSwiftServerClass(unittest.TestCase): def join_run_dir(self, path): return os.path.join(swift_init.RUN_DIR, path) - def test_server_init(self): + def test_create_server(self): server = swift_init.SwiftServer('proxy') self.assertEquals(server.server, 'proxy-server') self.assertEquals(server.type, 'proxy') @@ -219,6 +220,24 @@ class TestSwiftServerClass(unittest.TestCase): self.assertEquals(server.type, 'object') self.assertEquals(server.cmd, 'swift-object-replicator') + def test_server_to_string(self): + server = swift_init.SwiftServer('Proxy') + self.assertEquals(str(server), 'proxy-server') + server = swift_init.SwiftServer('object-replicator') + self.assertEquals(str(server), 'object-replicator') + + def test_server_repr(self): + server = swift_init.SwiftServer('proxy') + self.assert_(server.__class__.__name__ in repr(server)) + self.assert_(str(server) in repr(server)) + + def test_server_equality(self): + server1 = swift_init.SwiftServer('Proxy') + server2 = swift_init.SwiftServer('proxy-server') + self.assertEquals(server1, server2) + # it is NOT a string + self.assertNotEquals(server1, 'proxy-server') + def test_get_pid_file_name(self): server = swift_init.SwiftServer('proxy') ini_file = self.join_swift_dir('proxy-server.conf') @@ -1026,7 +1045,6 @@ class TestSwiftServerClass(unittest.TestCase): finally: sys.stdout = old_stdout - #TODO: more tests def test_stop(self): ini_files = ( 'account-server/1.conf', @@ -1082,18 +1100,193 @@ class TestSwiftServerClass(unittest.TestCase): self.assertFalse(os.path.exists(conf4)) self.assertFalse(os.path.exists(conf3)) -#TODO: test SwiftInit class class TestSwiftInitClass(unittest.TestCase): - def test_placeholder(self): - pass + def test_create(self): + controller = swift_init.SwiftInit(['test']) + self.assertEquals(len(controller.servers), 1) + server = controller.servers.pop() + self.assert_(isinstance(server, swift_init.SwiftServer)) + self.assertEquals(server.server, 'test-server') + # test multi-server and simple dedupe + servers = ['object-replicator', 'object-auditor', 'object-replicator'] + controller = swift_init.SwiftInit(servers) + self.assertEquals(len(controller.servers), 2) + for server in controller.servers: + self.assert_(server.server in servers) + # test all + controller = swift_init.SwiftInit(['all']) + self.assertEquals(len(controller.servers), len(swift_init.ALL_SERVERS)) + for server in controller.servers: + self.assert_(server.server in swift_init.ALL_SERVERS) + # test main + controller = swift_init.SwiftInit(['main']) + self.assertEquals(len(controller.servers), len(swift_init.MAIN_SERVERS)) + for server in controller.servers: + self.assert_(server.server in swift_init.MAIN_SERVERS) + # test rest + controller = swift_init.SwiftInit(['rest']) + self.assertEquals(len(controller.servers), len(swift_init.REST_SERVERS)) + for server in controller.servers: + self.assert_(server.server in swift_init.REST_SERVERS) + # test main + rest == all + controller = swift_init.SwiftInit(['main', 'rest']) + self.assertEquals(len(controller.servers), len(swift_init.ALL_SERVERS)) + for server in controller.servers: + self.assert_(server.server in swift_init.ALL_SERVERS) + # test dedupe + controller = swift_init.SwiftInit(['main', 'rest', 'proxy', 'object', + 'container', 'account']) + self.assertEquals(len(controller.servers), len(swift_init.ALL_SERVERS)) + for server in controller.servers: + self.assert_(server.server in swift_init.ALL_SERVERS) + + #TODO: more tests + def test_watch_server_pids(self): + raise SkipTest + + def test_get_command(self): + raise SkipTest + + def test_list_commands(self): + for cmd, help in swift_init.SwiftInit.list_commands(): + method = getattr(swift_init.SwiftInit, cmd.replace('-', '_'), None) + self.assert_(method, '%s is not a command' % cmd) + self.assert_(getattr(method, 'publicly_accessible', False)) + self.assertEquals(method.__doc__.strip(), help) + + def test_run_command(self): + raise SkipTest + + def test_status(self): + class MockSwiftServer(): + def __init__(self, server): + self.server = server + self.called_kwargs = [] + def status(self, **kwargs): + self.called_kwargs.append(kwargs) + if 'error' in self.server: + return 1 + else: + return 0 + + old_server_class = swift_init.SwiftServer + try: + swift_init.SwiftServer = MockSwiftServer + controller = swift_init.SwiftInit(['test']) + status = controller.status() + self.assertEquals(status, 0) + controller = swift_init.SwiftInit(['error']) + status = controller.status() + self.assertEquals(status, 1) + # test multi-server + controller = swift_init.SwiftInit(['test', 'error']) + kwargs = {'key': 'value'} + status = controller.status(**kwargs) + self.assertEquals(status, 1) + for server in controller.servers: + self.assertEquals(server.called_kwargs, [kwargs]) + finally: + swift_init.SwiftServer = old_server_class + + def test_start(self): + def mock_setup_env(): + getattr(mock_setup_env, 'called', []).append(True) + class MockSwiftServer(): + + def __init__(self, server): + self.server = server + self.called = defaultdict(list) + + def launch(self, **kwargs): + self.called['launch'].append(kwargs) + + def wait(self, **kwargs): + self.called['wait'].append(kwargs) + if 'error' in self.server: + return 1 + else: + return 0 + + def interact(self, **kwargs): + self.called['interact'].append(kwargs) + # TODO: test user quit + """ + if 'raise' in self.server: + raise KeyboardInterrupt + el + """ + if 'error' in self.server: + return 1 + else: + return 0 + + old_setup_env = swift_init.setup_env + old_swift_server = swift_init.SwiftServer + try: + swift_init.setup_env = mock_setup_env + swift_init.SwiftServer = MockSwiftServer + + # test no errors on launch + controller = swift_init.SwiftInit(['proxy', 'error']) + status = controller.start() + self.assertEquals(status, 0) + for server in controller.servers: + self.assertEquals(server.called['launch'], [{}]) + + # test wait + controller = swift_init.SwiftInit(['proxy', 'error']) + kwargs = {'wait': True} + status = controller.start(**kwargs) + self.assertEquals(status, 1) + for server in controller.servers: + self.assertEquals(server.called['launch'], [kwargs]) + self.assertEquals(server.called['wait'], [kwargs]) + + # test interact + controller = swift_init.SwiftInit(['proxy', 'error']) + kwargs = {'daemon': False} + status = controller.start(**kwargs) + self.assertEquals(status, 1) + for server in controller.servers: + self.assertEquals(server.called['launch'], [kwargs]) + self.assertEquals(server.called['interact'], [kwargs]) + finally: + swift_init.setup_env = old_setup_env + swift_init.SwiftServer = old_swift_server + + + def test_wait(self): + raise SkipTest + + def test_no_daemon(self): + raise SkipTest + + def test_once(self): + raise SkipTest + + def test_stop(self): + raise SkipTest + + def test_shutdown(self): + raise SkipTest + + def test_restart(self): + raise SkipTest + + def test_reload(self): + raise SkipTest + + def test_force_reload(self): + raise SkipTest + #TODO: test main class TestMain(unittest.TestCase): def test_placeholder(self): - pass + raise SkipTest if __name__ == '__main__': From 67de0c88f456a5bd8a812fc8cbfd7fad209a7ab4 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Sun, 16 Jan 2011 09:52:08 +0000 Subject: [PATCH 09/64] ipv6 support --- bin/swauth-add-account | 2 +- bin/swauth-add-user | 2 +- bin/swauth-delete-account | 2 +- bin/swauth-delete-user | 2 +- bin/swauth-list | 2 +- bin/swauth-prep | 2 +- bin/swauth-set-account-service | 2 +- bin/swift-ring-builder | 13 +++++++-- etc/proxy-server.conf-sample | 4 +-- swift/auth/server.py | 3 +- swift/common/bench.py | 3 +- swift/common/middleware/acl.py | 2 +- swift/common/middleware/swauth.py | 15 +++++----- swift/common/utils.py | 33 ++++++++++++++++++++++ swift/common/wsgi.py | 6 +++- swift/container/server.py | 2 +- swift/obj/server.py | 2 +- test/unit/common/middleware/test_swauth.py | 18 ++++++------ test/unit/common/test_utils.py | 21 ++++++++++++++ 19 files changed, 99 insertions(+), 37 deletions(-) diff --git a/bin/swauth-add-account b/bin/swauth-add-account index 32aceffc7b..fe18b5a72d 100755 --- a/bin/swauth-add-account +++ b/bin/swauth-add-account @@ -18,9 +18,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swauth-add-user b/bin/swauth-add-user index a844ed2a37..045dc0a766 100755 --- a/bin/swauth-add-user +++ b/bin/swauth-add-user @@ -18,9 +18,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swauth-delete-account b/bin/swauth-delete-account index c46e5e3b91..3d98f6ec4e 100755 --- a/bin/swauth-delete-account +++ b/bin/swauth-delete-account @@ -18,9 +18,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swauth-delete-user b/bin/swauth-delete-user index 5ee162437c..ede076dd5b 100755 --- a/bin/swauth-delete-user +++ b/bin/swauth-delete-user @@ -18,9 +18,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swauth-list b/bin/swauth-list index 7433e3ddfd..85a7633966 100755 --- a/bin/swauth-list +++ b/bin/swauth-list @@ -22,9 +22,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swauth-prep b/bin/swauth-prep index 5a931ae1d0..3d2cb7d3eb 100755 --- a/bin/swauth-prep +++ b/bin/swauth-prep @@ -18,9 +18,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swauth-set-account-service b/bin/swauth-set-account-service index 32eb06dc6b..054e4cfc4b 100755 --- a/bin/swauth-set-account-service +++ b/bin/swauth-set-account-service @@ -22,9 +22,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swift-ring-builder b/bin/swift-ring-builder index c448bea5ca..41293f7d37 100755 --- a/bin/swift-ring-builder +++ b/bin/swift-ring-builder @@ -235,10 +235,17 @@ swift-ring-builder add z-:/_ print 'Invalid add value: %s' % argv[3] exit(EXIT_ERROR) i = 1 - while i < len(rest) and rest[i] in '0123456789.': + if rest[i] == '[': + while i < len(rest) and rest[i] != ']': + i += 1 + ip = rest[2:i] i += 1 - ip = rest[1:i] - rest = rest[i:] + rest = rest[i:] + else: + while i < len(rest) and rest[i] in '0123456789.': + i += 1 + ip = rest[1:i] + rest = rest[i:] if not rest.startswith(':'): print 'Invalid add value: %s' % argv[3] diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index fda7d0d034..2d85f19508 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -68,13 +68,13 @@ use = egg:swift#swauth # auth_prefix = /auth/ # Cluster strings are of the format name:url where name is a short name for the # Swift cluster and url is the url to the proxy server(s) for the cluster. -# default_swift_cluster = local:http://127.0.0.1:8080/v1 +# default_swift_cluster = local#http://127.0.0.1:8080/v1 # You may also use the format name::url::url where the first url is the one # given to users to access their account (public url) and the second is the one # used by swauth itself to create and delete accounts (private url). This is # useful when a load balancer url should be used by users, but swauth itself is # behind the load balancer. Example: -# default_swift_cluster = local::https://public.com:8080/v1::http://private.com:8080/v1 +# default_swift_cluster = local##https://public.com:8080/v1##http://private.com:8080/v1 # token_life = 86400 # node_timeout = 10 # Highly recommended to change this. diff --git a/swift/auth/server.py b/swift/auth/server.py index a0bd31ccda..967f853291 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -20,7 +20,6 @@ from contextlib import contextmanager from time import gmtime, strftime, time from urllib import unquote, quote from uuid import uuid4 -from urlparse import urlparse from hashlib import md5, sha1 import hmac import base64 @@ -32,7 +31,7 @@ from webob.exc import HTTPBadRequest, HTTPConflict, HTTPForbidden, \ from swift.common.bufferedhttp import http_connect_raw as http_connect from swift.common.db import get_db_connection -from swift.common.utils import get_logger, split_path +from swift.common.utils import get_logger, split_path, urlparse class AuthController(object): diff --git a/swift/common/bench.py b/swift/common/bench.py index 4abafeb947..169497ef13 100644 --- a/swift/common/bench.py +++ b/swift/common/bench.py @@ -16,13 +16,12 @@ import uuid import time import random -from urlparse import urlparse from contextlib import contextmanager import eventlet.pools from eventlet.green.httplib import CannotSendRequest -from swift.common.utils import TRUE_VALUES +from swift.common.utils import TRUE_VALUES, urlparse from swift.common import client from swift.common import direct_client diff --git a/swift/common/middleware/acl.py b/swift/common/middleware/acl.py index f6784953ac..f08780eedb 100644 --- a/swift/common/middleware/acl.py +++ b/swift/common/middleware/acl.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from urlparse import urlparse +from swift.common.utils import urlparse def clean_acl(name, value): diff --git a/swift/common/middleware/swauth.py b/swift/common/middleware/swauth.py index 961f3a3ba4..568b00fb35 100644 --- a/swift/common/middleware/swauth.py +++ b/swift/common/middleware/swauth.py @@ -21,7 +21,6 @@ from httplib import HTTPConnection, HTTPSConnection from time import gmtime, strftime, time from traceback import format_exc from urllib import quote, unquote -from urlparse import urlparse from uuid import uuid4 from eventlet.timeout import Timeout @@ -32,7 +31,7 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ from swift.common.bufferedhttp import http_connect_raw as http_connect from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed -from swift.common.utils import cache_from_env, get_logger, split_path +from swift.common.utils import cache_from_env, get_logger, split_path, urlparse class Swauth(object): @@ -61,23 +60,23 @@ class Swauth(object): self.auth_prefix += '/' self.auth_account = '%s.auth' % self.reseller_prefix self.default_swift_cluster = conf.get('default_swift_cluster', - 'local:http://127.0.0.1:8080/v1') + 'local#http://127.0.0.1:8080/v1') # This setting is a little messy because of the options it has to # provide. The basic format is cluster_name:url, such as the default - # value of local:http://127.0.0.1:8080/v1. But, often the url given to + # value of local#http://127.0.0.1:8080/v1. But, often the url given to # the user needs to be different than the url used by Swauth to # create/delete accounts. So there's a more complex format of # cluster_name::url::url, such as - # local::https://public.com:8080/v1::http://private.com:8080/v1. + # local##https://public.com:8080/v1##http://private.com:8080/v1. # The double colon is what sets the two apart. - if '::' in self.default_swift_cluster: + if '##' in self.default_swift_cluster: self.dsc_name, self.dsc_url, self.dsc_url2 = \ - self.default_swift_cluster.split('::', 2) + self.default_swift_cluster.split('##', 2) self.dsc_url = self.dsc_url.rstrip('/') self.dsc_url2 = self.dsc_url2.rstrip('/') else: self.dsc_name, self.dsc_url = \ - self.default_swift_cluster.split(':', 1) + self.default_swift_cluster.split('#', 1) self.dsc_url = self.dsc_url2 = self.dsc_url.rstrip('/') self.dsc_parsed = urlparse(self.dsc_url) if self.dsc_parsed.scheme not in ('http', 'https'): diff --git a/swift/common/utils.py b/swift/common/utils.py index 299980493a..05b15e99fa 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -34,6 +34,7 @@ from ConfigParser import ConfigParser, NoSectionError, NoOptionError from optparse import OptionParser from tempfile import mkstemp import cPickle as pickle +from urlparse import urlparse as stdlib_urlparse, ParseResult import eventlet from eventlet import greenio, GreenPool, sleep, Timeout, listen @@ -845,3 +846,35 @@ def ratelimit_sleep(running_time, max_rate, incr_by=1): elif running_time - now > time_per_request: eventlet.sleep((running_time - now) / clock_accuracy) return running_time + time_per_request + + +class ModifiedParseResult(ParseResult): + "Parse results class for urlparse." + + @property + def hostname(self): + netloc = self.netloc.split('@', 1)[-1] + if netloc.startswith('['): + return netloc[1:].split(']')[0] + elif ':' in netloc: + return netloc.rsplit(':')[0] + return netloc + + @property + def port(self): + netloc = self.netloc.split('@', 1)[-1] + if netloc.startswith('['): + netloc = netloc.rsplit(']')[1] + if ':' in netloc: + return int(netloc.rsplit(':')[1]) + return None + + +def urlparse(url): + """ + urlparse augmentation. + This is necessary because urlparse can't handle RFC 2732 URLs. + + :param url: URL to parse. + """ + return ModifiedParseResult(*stdlib_urlparse(url)) diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 9450bcf439..cedc4b2c8b 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -68,11 +68,15 @@ def get_socket(conf, default_port=8080): """ bind_addr = (conf.get('bind_ip', '0.0.0.0'), int(conf.get('bind_port', default_port))) + address_family = [addr[0] for addr in socket.getaddrinfo(bind_addr[0], + bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM) + if addr[0] in (socket.AF_INET, socket.AF_INET6)][0] sock = None retry_until = time.time() + 30 while not sock and time.time() < retry_until: try: - sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096))) + sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096)), + family=address_family) if 'cert_file' in conf: sock = ssl.wrap_socket(sock, certfile=conf['cert_file'], keyfile=conf['key_file']) diff --git a/swift/container/server.py b/swift/container/server.py index 7ba375ce33..1ffba8a909 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -88,7 +88,7 @@ class ContainerController(object): account_partition = req.headers.get('X-Account-Partition') account_device = req.headers.get('X-Account-Device') if all([account_host, account_partition, account_device]): - account_ip, account_port = account_host.split(':') + account_ip, account_port = account_host.rsplit(':', 1) new_path = '/' + '/'.join([account, container]) info = broker.get_info() account_headers = {'x-put-timestamp': info['put_timestamp'], diff --git a/swift/obj/server.py b/swift/obj/server.py index 4afc38057d..f20b40d57a 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -294,7 +294,7 @@ class ObjectController(object): full_path = '/%s/%s/%s' % (account, container, obj) try: with ConnectionTimeout(self.conn_timeout): - ip, port = host.split(':') + ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): diff --git a/test/unit/common/middleware/test_swauth.py b/test/unit/common/middleware/test_swauth.py index 00c010b9dc..a6edab9c2c 100644 --- a/test/unit/common/middleware/test_swauth.py +++ b/test/unit/common/middleware/test_swauth.py @@ -151,21 +151,21 @@ class TestAuth(unittest.TestCase): app = FakeApp() self.assertRaises(Exception, auth.filter_factory({ 'super_admin_key': 'supertest', - 'default_swift_cluster': 'local:badscheme://host/path'}), app) + 'default_swift_cluster': 'local#badscheme://host/path'}), app) ath = auth.filter_factory({'super_admin_key': 'supertest'})(app) self.assertEquals(ath.default_swift_cluster, - 'local:http://127.0.0.1:8080/v1') + 'local#http://127.0.0.1:8080/v1') ath = auth.filter_factory({'super_admin_key': 'supertest', - 'default_swift_cluster': 'local:http://host/path'})(app) + 'default_swift_cluster': 'local#http://host/path'})(app) self.assertEquals(ath.default_swift_cluster, - 'local:http://host/path') + 'local#http://host/path') ath = auth.filter_factory({'super_admin_key': 'supertest', - 'default_swift_cluster': 'local:https://host/path/'})(app) + 'default_swift_cluster': 'local#https://host/path/'})(app) self.assertEquals(ath.dsc_url, 'https://host/path') self.assertEquals(ath.dsc_url2, 'https://host/path') ath = auth.filter_factory({'super_admin_key': 'supertest', 'default_swift_cluster': - 'local::https://host/path/::http://host2/path2/'})(app) + 'local##https://host/path/##http://host2/path2/'})(app) self.assertEquals(ath.dsc_url, 'https://host/path') self.assertEquals(ath.dsc_url2, 'http://host2/path2') @@ -2882,7 +2882,7 @@ class TestAuth(unittest.TestCase): def test_get_conn_default_https(self): local_auth = auth.filter_factory({'super_admin_key': 'supertest', - 'default_swift_cluster': 'local:https://1.2.3.4/v1'})(FakeApp()) + 'default_swift_cluster': 'local#https://1.2.3.4/v1'})(FakeApp()) conn = local_auth.get_conn() self.assertEquals(conn.__class__, auth.HTTPSConnection) self.assertEquals(conn.host, '1.2.3.4') @@ -2890,7 +2890,7 @@ class TestAuth(unittest.TestCase): def test_get_conn_overridden(self): local_auth = auth.filter_factory({'super_admin_key': 'supertest', - 'default_swift_cluster': 'local:https://1.2.3.4/v1'})(FakeApp()) + 'default_swift_cluster': 'local#https://1.2.3.4/v1'})(FakeApp()) conn = \ local_auth.get_conn(urlparsed=auth.urlparse('http://5.6.7.8/v1')) self.assertEquals(conn.__class__, auth.HTTPConnection) @@ -2899,7 +2899,7 @@ class TestAuth(unittest.TestCase): def test_get_conn_overridden_https(self): local_auth = auth.filter_factory({'super_admin_key': 'supertest', - 'default_swift_cluster': 'local:http://1.2.3.4/v1'})(FakeApp()) + 'default_swift_cluster': 'local#http://1.2.3.4/v1'})(FakeApp()) conn = \ local_auth.get_conn(urlparsed=auth.urlparse('https://5.6.7.8/v1')) self.assertEquals(conn.__class__, auth.HTTPSConnection) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 1f5a94edd5..b9e8a3f81b 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -477,6 +477,27 @@ log_name = yarr''' total += i self.assertTrue(abs(50 - (time.time() - start) * 100) < 10) + def test_urlparse(self): + parsed = utils.urlparse('http://127.0.0.1/') + self.assertEquals(parsed.scheme, 'http') + self.assertEquals(parsed.hostname, '127.0.0.1') + self.assertEquals(parsed.path, '/') + + parsed = utils.urlparse('http://127.0.0.1:8080/') + self.assertEquals(parsed.port, 8080) + + parsed = utils.urlparse('https://127.0.0.1/') + self.assertEquals(parsed.scheme, 'https') + + parsed = utils.urlparse('http://[::1]/') + self.assertEquals(parsed.hostname, '::1') + + parsed = utils.urlparse('http://[::1]:8080/') + self.assertEquals(parsed.hostname, '::1') + self.assertEquals(parsed.port, 8080) + + parsed = utils.urlparse('www.example.com') + self.assertEquals(parsed.hostname, '') if __name__ == '__main__': unittest.main() From 37ca9e569a828669db11cc0eaad7063d2c748a7c Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 19 Jan 2011 23:43:23 -0600 Subject: [PATCH 10/64] more tests, still slow going --- bin/swift-init | 6 +- test/bin/test_swift_init.py | 129 ++++++++++++++++++++++++++++++++++-- 2 files changed, 128 insertions(+), 7 deletions(-) diff --git a/bin/swift-init b/bin/swift-init index cd56e412be..ce72242c0d 100755 --- a/bin/swift-init +++ b/bin/swift-init @@ -227,18 +227,18 @@ class SwiftInit(): return status @command - def wait(self, daemon=True, **kwargs): + def wait(self, **kwargs): """spawn server and wait for it to start """ kwargs['wait'] = True - self.start(**kwargs) + return self.start(**kwargs) @command def no_daemon(self, **kwargs): """start a server interactivly """ kwargs['daemon'] = False - self.start(**kwargs) + return self.start(**kwargs) @command def once(self, **kwargs): diff --git a/test/bin/test_swift_init.py b/test/bin/test_swift_init.py index 0d141f15b2..fb623c3ec7 100644 --- a/test/bin/test_swift_init.py +++ b/test/bin/test_swift_init.py @@ -1234,7 +1234,7 @@ class TestSwiftInitClass(unittest.TestCase): for server in controller.servers: self.assertEquals(server.called['launch'], [{}]) - # test wait + # test error on wait controller = swift_init.SwiftInit(['proxy', 'error']) kwargs = {'wait': True} status = controller.start(**kwargs) @@ -1257,13 +1257,134 @@ class TestSwiftInitClass(unittest.TestCase): def test_wait(self): - raise SkipTest + class MockSwiftServer(): + def __init__(self, server): + self.server = server + self.called = defaultdict(list) + + def launch(self, **kwargs): + self.called['launch'].append(kwargs) + + def wait(self, **kwargs): + self.called['wait'].append(kwargs) + return int('error' in self.server) + + orig_swift_server = swift_init.SwiftServer + try: + swift_init.SwiftServer = MockSwiftServer + # test success + init = swift_init.SwiftInit(['proxy']) + status = init.wait() + self.assertEquals(status, 0) + for server in init.servers: + self.assertEquals(len(server.called['launch']), 1) + called_kwargs = server.called['launch'][0] + self.assert_('wait' in called_kwargs) + self.assert_(called_kwargs['wait']) + self.assertEquals(len(server.called['wait']), 1) + called_kwargs = server.called['wait'][0] + self.assert_('wait' in called_kwargs) + self.assert_(called_kwargs['wait']) + # test error + init = swift_init.SwiftInit(['error']) + status = init.wait() + self.assertEquals(status, 1) + for server in init.servers: + self.assertEquals(len(server.called['launch']), 1) + called_kwargs = server.called['launch'][0] + self.assert_('wait' in called_kwargs) + self.assert_(called_kwargs['wait']) + self.assertEquals(len(server.called['wait']), 1) + called_kwargs = server.called['wait'][0] + self.assert_('wait' in called_kwargs) + self.assert_(called_kwargs['wait']) + # test wait with once option + init = swift_init.SwiftInit(['updater', 'replicator-error']) + status = init.wait(once=True) + self.assertEquals(status, 1) + for server in init.servers: + self.assertEquals(len(server.called['launch']), 1) + called_kwargs = server.called['launch'][0] + self.assert_('wait' in called_kwargs) + self.assert_(called_kwargs['wait']) + self.assert_('once' in called_kwargs) + self.assert_(called_kwargs['once']) + self.assertEquals(len(server.called['wait']), 1) + called_kwargs = server.called['wait'][0] + self.assert_('wait' in called_kwargs) + self.assert_(called_kwargs['wait']) + self.assert_('once' in called_kwargs) + self.assert_(called_kwargs['once']) + finally: + swift_init.SwiftServer = orig_swift_server + def test_no_daemon(self): - raise SkipTest + class MockSwiftServer(): + + def __init__(self, server): + self.server = server + self.called = defaultdict(list) + + def launch(self, **kwargs): + self.called['launch'].append(kwargs) + + def interact(self, **kwargs): + self.called['interact'].append(kwargs) + return int('error' in self.server) + + orig_swift_server = swift_init.SwiftServer + try: + swift_init.SwiftServer = MockSwiftServer + # test success + init = swift_init.SwiftInit(['proxy']) + stats = init.no_daemon() + self.assertEquals(stats, 0) + # test error + init = swift_init.SwiftInit(['proxy', 'object-error']) + stats = init.no_daemon() + self.assertEquals(stats, 1) + # test once + init = swift_init.SwiftInit(['proxy', 'object-error']) + stats = init.no_daemon() + for server in init.servers: + self.assertEquals(len(server.called['launch']), 1) + self.assertEquals(len(server.called['wait']), 0) + self.assertEquals(len(server.called['interact']), 1) + finally: + swift_init.SwiftServer = orig_swift_server def test_once(self): - raise SkipTest + class MockSwiftServer(): + + def __init__(self, server): + self.server = server + self.called = defaultdict(list) + + def launch(self, **kwargs): + return self.called['launch'].append(kwargs) + + + orig_swift_server = swift_init.SwiftServer + try: + swift_init.SwiftServer = MockSwiftServer + # test no errors + init = swift_init.SwiftInit(['account-reaper']) + status = init.once() + self.assertEquals(status, 0) + # test no error code on error + init = swift_init.SwiftInit(['error-reaper']) + status = init.once() + self.assertEquals(status, 0) + for server in init.servers: + self.assertEquals(len(server.called['launch']), 1) + called_kwargs = server.called['launch'][0] + self.assertEquals(called_kwargs, {'once': True}) + self.assertEquals(len(server.called['wait']), 0) + self.assertEquals(len(server.called['interact']), 0) + finally: + swift_init.SwiftServer = orig_swift_server + def test_stop(self): raise SkipTest From a44635ca9767d8ee737e0189063cf3b5b3842285 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Sat, 29 Jan 2011 00:54:12 +0000 Subject: [PATCH 11/64] support WAL journaling instead of .pending files --- swift/common/db.py | 195 +++++++-------------------------------------- 1 file changed, 29 insertions(+), 166 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index be96411619..36ef1f3c91 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -33,7 +33,7 @@ import simplejson as json import sqlite3 from swift.common.utils import normalize_timestamp, renamer, \ - mkdirs, lock_parent_directory, fallocate + mkdirs, lock_parent_directory from swift.common.exceptions import LockTimeout @@ -41,8 +41,7 @@ from swift.common.exceptions import LockTimeout BROKER_TIMEOUT = 25 #: Pickle protocol to use PICKLE_PROTOCOL = 2 -#: Max number of pending entries -PENDING_CAP = 131072 +PENDING_COMMIT_TIMEOUT = 900 class DatabaseConnectionError(sqlite3.DatabaseError): @@ -139,7 +138,7 @@ def get_db_connection(path, timeout=30, okay_to_create=False): conn.execute('PRAGMA synchronous = NORMAL') conn.execute('PRAGMA count_changes = OFF') conn.execute('PRAGMA temp_store = MEMORY') - conn.execute('PRAGMA journal_mode = DELETE') + conn.execute('PRAGMA journal_mode = WAL') conn.create_function('chexor', 3, chexor) except sqlite3.DatabaseError: import traceback @@ -152,13 +151,10 @@ class DatabaseBroker(object): """Encapsulates working with a database.""" def __init__(self, db_file, timeout=BROKER_TIMEOUT, logger=None, - account=None, container=None, pending_timeout=10, - stale_reads_ok=False): + account=None, container=None, stale_reads_ok=False): """ Encapsulates working with a database. """ self.conn = None self.db_file = db_file - self.pending_file = self.db_file + '.pending' - self.pending_timeout = pending_timeout self.stale_reads_ok = stale_reads_ok self.db_dir = os.path.dirname(db_file) self.timeout = timeout @@ -233,7 +229,7 @@ class DatabaseBroker(object): conn.close() with open(tmp_db_file, 'r+b') as fp: os.fsync(fp.fileno()) - with lock_parent_directory(self.db_file, self.pending_timeout): + with lock_parent_directory(self.db_file, self.timeout): if os.path.exists(self.db_file): # It's as if there was a "condition" where different parts # of the system were "racing" each other. @@ -348,11 +344,6 @@ class DatabaseBroker(object): :param count: number to get :returns: list of objects between start and end """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise with self.get() as conn: curs = conn.execute(''' SELECT * FROM %s WHERE ROWID > ? ORDER BY ROWID ASC LIMIT ? @@ -401,11 +392,7 @@ class DatabaseBroker(object): :returns: dict containing keys: hash, id, created_at, put_timestamp, delete_timestamp, count, max_row, and metadata """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise + self._commit_puts() query_part1 = ''' SELECT hash, id, created_at, put_timestamp, delete_timestamp, %s_count AS count, @@ -455,34 +442,6 @@ class DatabaseBroker(object): (rec['sync_point'], rec['remote_id'])) conn.commit() - def _preallocate(self): - """ - The idea is to allocate space in front of an expanding db. If it gets - within 512k of a boundary, it allocates to the next boundary. - Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after. - """ - if self.db_file == ':memory:': - return - MB = (1024 * 1024) - - def prealloc_points(): - for pm in (1, 2, 5, 10, 25, 50): - yield pm * MB - while True: - pm += 50 - yield pm * MB - - stat = os.stat(self.db_file) - file_size = stat.st_size - allocated_size = stat.st_blocks * 512 - for point in prealloc_points(): - if file_size <= point - MB / 2: - prealloc_size = point - break - if allocated_size < prealloc_size: - with open(self.db_file, 'rb+') as fp: - fallocate(fp.fileno(), int(prealloc_size)) - @property def metadata(self): """ @@ -717,11 +676,6 @@ class ContainerBroker(DatabaseBroker): :returns: True if the database has no active objects, False otherwise """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise with self.get() as conn: row = conn.execute( 'SELECT object_count from container_stat').fetchone() @@ -729,17 +683,16 @@ class ContainerBroker(DatabaseBroker): def _commit_puts(self, item_list=None): """Handles commiting rows in .pending files.""" - if self.db_file == ':memory:' or not os.path.exists(self.pending_file): + pending_file = self.db_file + '.pending' + if self.db_file == ':memory:' or not os.path.exists(pending_file): + return + if not os.path.getsize(pending_file): + os.unlink(pending_file) return if item_list is None: item_list = [] - with lock_parent_directory(self.pending_file, self.pending_timeout): - self._preallocate() - if not os.path.getsize(self.pending_file): - if item_list: - self.merge_items(item_list) - return - with open(self.pending_file, 'r+b') as fp: + with lock_parent_directory(pending_file, PENDING_COMMIT_TIMEOUT): + with open(pending_file, 'r+b') as fp: for entry in fp.read().split(':'): if entry: try: @@ -752,11 +705,11 @@ class ContainerBroker(DatabaseBroker): except Exception: self.logger.exception( _('Invalid pending entry %(file)s: %(entry)s'), - {'file': self.pending_file, 'entry': entry}) + {'file': pending_file, 'entry': entry}) if item_list: self.merge_items(item_list) try: - os.ftruncate(fp.fileno(), 0) + os.unlink(pending_file) except OSError, err: if err.errno != errno.ENOENT: raise @@ -774,7 +727,6 @@ class ContainerBroker(DatabaseBroker): delete :param sync_timestamp: max update_at timestamp of sync rows to delete """ - self._commit_puts() with self.get() as conn: conn.execute(""" DELETE FROM object @@ -818,30 +770,9 @@ class ContainerBroker(DatabaseBroker): record = {'name': name, 'created_at': timestamp, 'size': size, 'content_type': content_type, 'etag': etag, 'deleted': deleted} - if self.db_file == ':memory:': - self.merge_items([record]) - return - if not os.path.exists(self.db_file): + if self.db_file != ':memory:' and not os.path.exists(self.db_file): raise DatabaseConnectionError(self.db_file, "DB doesn't exist") - pending_size = 0 - try: - pending_size = os.path.getsize(self.pending_file) - except OSError, err: - if err.errno != errno.ENOENT: - raise - if pending_size > PENDING_CAP: - self._commit_puts([record]) - else: - with lock_parent_directory( - self.pending_file, self.pending_timeout): - with open(self.pending_file, 'a+b') as fp: - # Colons aren't used in base64 encoding; so they are our - # delimiter - fp.write(':') - fp.write(pickle.dumps( - (name, timestamp, size, content_type, etag, deleted), - protocol=PICKLE_PROTOCOL).encode('base64')) - fp.flush() + self.merge_items([record]) def is_deleted(self, timestamp=None): """ @@ -851,11 +782,6 @@ class ContainerBroker(DatabaseBroker): """ if self.db_file != ':memory:' and not os.path.exists(self.db_file): return True - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise with self.get() as conn: row = conn.execute(''' SELECT put_timestamp, delete_timestamp, object_count @@ -878,11 +804,6 @@ class ContainerBroker(DatabaseBroker): reported_put_timestamp, reported_delete_timestamp, reported_object_count, reported_bytes_used, hash, id) """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise with self.get() as conn: return conn.execute(''' SELECT account, container, created_at, put_timestamp, @@ -919,11 +840,6 @@ class ContainerBroker(DatabaseBroker): :returns: list of object names """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise rv = [] with self.get() as conn: row = conn.execute(''' @@ -960,11 +876,6 @@ class ContainerBroker(DatabaseBroker): :returns: list of tuples of (name, created_at, size, content_type, etag) """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise if path is not None: prefix = path if path: @@ -1193,17 +1104,16 @@ class AccountBroker(DatabaseBroker): def _commit_puts(self, item_list=None): """Handles commiting rows in .pending files.""" - if self.db_file == ':memory:' or not os.path.exists(self.pending_file): + pending_file = self.db_file + '.pending' + if self.db_file == ':memory:' or not os.path.exists(pending_file): + return + if not os.path.getsize(pending_file): + os.unlink(pending_file) return if item_list is None: item_list = [] - with lock_parent_directory(self.pending_file, self.pending_timeout): - self._preallocate() - if not os.path.getsize(self.pending_file): - if item_list: - self.merge_items(item_list) - return - with open(self.pending_file, 'r+b') as fp: + with lock_parent_directory(pending_file, PENDING_COMMIT_TIMEOUT): + with open(pending_file, 'r+b') as fp: for entry in fp.read().split(':'): if entry: try: @@ -1219,11 +1129,11 @@ class AccountBroker(DatabaseBroker): except Exception: self.logger.exception( _('Invalid pending entry %(file)s: %(entry)s'), - {'file': self.pending_file, 'entry': entry}) + {'file': pending_file, 'entry': entry}) if item_list: self.merge_items(item_list) try: - os.ftruncate(fp.fileno(), 0) + os.unlink(pending_file) except OSError, err: if err.errno != errno.ENOENT: raise @@ -1234,11 +1144,6 @@ class AccountBroker(DatabaseBroker): :returns: True if the database has no active containers. """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise with self.get() as conn: row = conn.execute( 'SELECT container_count from account_stat').fetchone() @@ -1258,7 +1163,6 @@ class AccountBroker(DatabaseBroker): :param sync_timestamp: max update_at timestamp of sync rows to delete """ - self._commit_puts() with self.get() as conn: conn.execute(''' DELETE FROM container WHERE @@ -1286,11 +1190,6 @@ class AccountBroker(DatabaseBroker): :returns: put_timestamp of the container """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise with self.get() as conn: ret = conn.execute(''' SELECT put_timestamp FROM container @@ -1311,6 +1210,8 @@ class AccountBroker(DatabaseBroker): :param object_count: number of objects in the container :param bytes_used: number of bytes used by the container """ + if self.db_file != ':memory:' and not os.path.exists(self.db_file): + raise DatabaseConnectionError(self.db_file, "DB doesn't exist") if delete_timestamp > put_timestamp and \ object_count in (None, '', 0, '0'): deleted = 1 @@ -1321,24 +1222,7 @@ class AccountBroker(DatabaseBroker): 'object_count': object_count, 'bytes_used': bytes_used, 'deleted': deleted} - if self.db_file == ':memory:': - self.merge_items([record]) - return - commit = False - with lock_parent_directory(self.pending_file, self.pending_timeout): - with open(self.pending_file, 'a+b') as fp: - # Colons aren't used in base64 encoding; so they are our - # delimiter - fp.write(':') - fp.write(pickle.dumps( - (name, put_timestamp, delete_timestamp, object_count, - bytes_used, deleted), - protocol=PICKLE_PROTOCOL).encode('base64')) - fp.flush() - if fp.tell() > PENDING_CAP: - commit = True - if commit: - self._commit_puts() + self.merge_items([record]) def can_delete_db(self, cutoff): """ @@ -1346,7 +1230,6 @@ class AccountBroker(DatabaseBroker): :returns: True if the account can be deleted, False otherwise """ - self._commit_puts() with self.get() as conn: row = conn.execute(''' SELECT status, put_timestamp, delete_timestamp, container_count @@ -1372,11 +1255,6 @@ class AccountBroker(DatabaseBroker): """ if self.db_file != ':memory:' and not os.path.exists(self.db_file): return True - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise with self.get() as conn: row = conn.execute(''' SELECT put_timestamp, delete_timestamp, container_count, status @@ -1401,11 +1279,6 @@ class AccountBroker(DatabaseBroker): delete_timestamp, container_count, object_count, bytes_used, hash, id) """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise with self.get() as conn: return conn.execute(''' SELECT account, created_at, put_timestamp, delete_timestamp, @@ -1422,11 +1295,6 @@ class AccountBroker(DatabaseBroker): :returns: list of container names """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise rv = [] with self.get() as conn: row = conn.execute(''' @@ -1460,11 +1328,6 @@ class AccountBroker(DatabaseBroker): :returns: list of tuples of (name, object_count, bytes_used, 0) """ - try: - self._commit_puts() - except LockTimeout: - if not self.stale_reads_ok: - raise if delimiter and not prefix: prefix = '' orig_marker = marker From 4e100f6b325cbc5b2d83b4f3b622636ca25b069d Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Sat, 29 Jan 2011 01:23:18 +0000 Subject: [PATCH 12/64] retry connect refactor --- swift/common/db.py | 49 +++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index 36ef1f3c91..ca667edf1a 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -27,6 +27,7 @@ import cPickle as pickle import errno from random import randint from tempfile import mkstemp +import traceback from eventlet import sleep import simplejson as json @@ -41,6 +42,7 @@ from swift.common.exceptions import LockTimeout BROKER_TIMEOUT = 25 #: Pickle protocol to use PICKLE_PROTOCOL = 2 +CONNECT_ATTEMPTS = 4 PENDING_COMMIT_TIMEOUT = 900 @@ -122,29 +124,32 @@ def get_db_connection(path, timeout=30, okay_to_create=False): :param okay_to_create: if True, create the DB if it doesn't exist :returns: DB connection object """ - try: - connect_time = time.time() - conn = sqlite3.connect(path, check_same_thread=False, - factory=GreenDBConnection, timeout=timeout) - if path != ':memory:' and not okay_to_create: + # retry logic to address: + # http://www.mail-archive.com/sqlite-users@sqlite.org/msg57092.html + for tries in xrange(1, CONNECT_ATTEMPTS + 1): + try: + connect_time = time.time() + conn = sqlite3.connect(path, check_same_thread=False, + factory=GreenDBConnection, timeout=timeout) # attempt to detect and fail when connect creates the db file - stat = os.stat(path) - if stat.st_size == 0 and stat.st_ctime >= connect_time: - os.unlink(path) - raise DatabaseConnectionError(path, - 'DB file created by connect?') - conn.row_factory = sqlite3.Row - conn.text_factory = str - conn.execute('PRAGMA synchronous = NORMAL') - conn.execute('PRAGMA count_changes = OFF') - conn.execute('PRAGMA temp_store = MEMORY') - conn.execute('PRAGMA journal_mode = WAL') - conn.create_function('chexor', 3, chexor) - except sqlite3.DatabaseError: - import traceback - raise DatabaseConnectionError(path, traceback.format_exc(), - timeout=timeout) - return conn + if path != ':memory:' and not okay_to_create: + stat = os.stat(path) + if stat.st_size == 0 and stat.st_ctime >= connect_time: + os.unlink(path) + raise DatabaseConnectionError(path, + 'DB file created by connect?') + conn.execute('PRAGMA synchronous = NORMAL') + conn.execute('PRAGMA count_changes = OFF') + conn.execute('PRAGMA temp_store = MEMORY') + conn.execute('PRAGMA journal_mode = WAL') + conn.create_function('chexor', 3, chexor) + conn.row_factory = sqlite3.Row + conn.text_factory = str + return conn + except sqlite3.DatabaseError, e: + if tries == CONNECT_ATTEMPTS or 'locking protocol' not in str(e): + raise DatabaseConnectionError(path, traceback.format_exc(), + timeout=timeout) class DatabaseBroker(object): From d83ce428afec5af180e5f85104da6242f8801fc1 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Sat, 29 Jan 2011 01:40:55 +0000 Subject: [PATCH 13/64] increase WAL autocheckpoint --- swift/common/db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/swift/common/db.py b/swift/common/db.py index ca667edf1a..e06739d85f 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -44,6 +44,7 @@ BROKER_TIMEOUT = 25 PICKLE_PROTOCOL = 2 CONNECT_ATTEMPTS = 4 PENDING_COMMIT_TIMEOUT = 900 +AUTOCHECKPOINT = 8192 class DatabaseConnectionError(sqlite3.DatabaseError): @@ -142,6 +143,7 @@ def get_db_connection(path, timeout=30, okay_to_create=False): conn.execute('PRAGMA count_changes = OFF') conn.execute('PRAGMA temp_store = MEMORY') conn.execute('PRAGMA journal_mode = WAL') + conn.execute('PRAGMA wal_autocheckpoint = %s' % AUTOCHECKPOINT) conn.create_function('chexor', 3, chexor) conn.row_factory = sqlite3.Row conn.text_factory = str From 0649d9cc602baaacdd428e3455df4f6a9254e681 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Sat, 29 Jan 2011 03:26:26 +0000 Subject: [PATCH 14/64] replication fixes for WAL --- swift/common/db.py | 2 ++ swift/common/db_replicator.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/swift/common/db.py b/swift/common/db.py index e06739d85f..0f288b74a1 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -288,6 +288,7 @@ class DatabaseBroker(object): self.conn = None orig_isolation_level = conn.isolation_level conn.isolation_level = None + conn.execute('PRAGMA journal_mode = DELETE') # remove any journal files conn.execute('BEGIN IMMEDIATE') try: yield True @@ -295,6 +296,7 @@ class DatabaseBroker(object): pass try: conn.execute('ROLLBACK') + conn.execute('PRAGMA journal_mode = WAL') # back to WAL mode conn.isolation_level = orig_isolation_level self.conn = conn except Exception: diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 49756f1f7b..01a7d202de 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -180,7 +180,9 @@ class Replicator(Daemon): return False # perform block-level sync if the db was modified during the first sync if os.path.exists(broker.db_file + '-journal') or \ - os.path.getmtime(broker.db_file) > mtime: + os.path.exists(broker.db_file + '-wal') or \ + os.path.exists(broker.db_file + '-shm') or \ + os.path.getmtime(broker.db_file) > mtime: # grab a lock so nobody else can modify it with broker.lock(): if not self._rsync_file(broker.db_file, remote_file, False): From 6f5d69e7b5b52f9c407e0ca83231778a42a48722 Mon Sep 17 00:00:00 2001 From: Jay Payne Date: Sat, 29 Jan 2011 16:43:02 +0000 Subject: [PATCH 15/64] listing is a tuple correcting reference rv[1] --- swift/common/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/common/client.py b/swift/common/client.py index bf402adb76..1fffaa493d 100644 --- a/swift/common/client.py +++ b/swift/common/client.py @@ -222,7 +222,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, listing = \ get_account(url, token, marker, limit, prefix, http_conn)[1] if listing: - rv.extend(listing) + rv[1].extend(listing) return rv parsed, conn = http_conn qs = 'format=json' From 68cda9b72446df358120bd9fed00a8804e960375 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Sat, 29 Jan 2011 18:22:16 +0000 Subject: [PATCH 16/64] refactor db open retry loop slightly --- swift/common/db.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index 0f288b74a1..1e0057908c 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -127,7 +127,7 @@ def get_db_connection(path, timeout=30, okay_to_create=False): """ # retry logic to address: # http://www.mail-archive.com/sqlite-users@sqlite.org/msg57092.html - for tries in xrange(1, CONNECT_ATTEMPTS + 1): + for attempt in xrange(CONNECT_ATTEMPTS): try: connect_time = time.time() conn = sqlite3.connect(path, check_same_thread=False, @@ -139,19 +139,18 @@ def get_db_connection(path, timeout=30, okay_to_create=False): os.unlink(path) raise DatabaseConnectionError(path, 'DB file created by connect?') + conn.execute('PRAGMA journal_mode = WAL') conn.execute('PRAGMA synchronous = NORMAL') + conn.execute('PRAGMA wal_autocheckpoint = %s' % AUTOCHECKPOINT) conn.execute('PRAGMA count_changes = OFF') conn.execute('PRAGMA temp_store = MEMORY') - conn.execute('PRAGMA journal_mode = WAL') - conn.execute('PRAGMA wal_autocheckpoint = %s' % AUTOCHECKPOINT) conn.create_function('chexor', 3, chexor) conn.row_factory = sqlite3.Row conn.text_factory = str return conn except sqlite3.DatabaseError, e: - if tries == CONNECT_ATTEMPTS or 'locking protocol' not in str(e): - raise DatabaseConnectionError(path, traceback.format_exc(), - timeout=timeout) + errstr = traceback.format_exc() + raise DatabaseConnectionError(path, errstr, timeout=timeout) class DatabaseBroker(object): From 625255da39d0dda986c47f4390343513a34e5943 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Sat, 29 Jan 2011 19:26:06 +0000 Subject: [PATCH 17/64] remove pending_timeout references --- swift/account/server.py | 7 ------- swift/common/db.py | 3 +-- swift/common/db_replicator.py | 2 +- swift/container/server.py | 4 ---- 4 files changed, 2 insertions(+), 14 deletions(-) diff --git a/swift/account/server.py b/swift/account/server.py index 2c83f51cc6..94399eec22 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -86,8 +86,6 @@ class AccountController(object): return Response(status='507 %s is not mounted' % drive) broker = self._get_account_broker(drive, part, account) if container: # put account container - if 'x-cf-trans-id' in req.headers: - broker.pending_timeout = 3 if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): return HTTPNotFound(request=req) @@ -140,9 +138,6 @@ class AccountController(object): if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) broker = self._get_account_broker(drive, part, account) - if not container: - broker.pending_timeout = 0.1 - broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() @@ -171,8 +166,6 @@ class AccountController(object): if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) broker = self._get_account_broker(drive, part, account) - broker.pending_timeout = 0.1 - broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() diff --git a/swift/common/db.py b/swift/common/db.py index 1e0057908c..7040b2446a 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -157,11 +157,10 @@ class DatabaseBroker(object): """Encapsulates working with a database.""" def __init__(self, db_file, timeout=BROKER_TIMEOUT, logger=None, - account=None, container=None, stale_reads_ok=False): + account=None, container=None): """ Encapsulates working with a database. """ self.conn = None self.db_file = db_file - self.stale_reads_ok = stale_reads_ok self.db_dir = os.path.dirname(db_file) self.timeout = timeout self.logger = logger or logging.getLogger() diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 01a7d202de..5c4d4ebd8e 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -318,7 +318,7 @@ class Replicator(Daemon): self.logger.debug(_('Replicating db %s'), object_file) self.stats['attempted'] += 1 try: - broker = self.brokerclass(object_file, pending_timeout=30) + broker = self.brokerclass(object_file) broker.reclaim(time.time() - self.reclaim_age, time.time() - (self.reclaim_age * 2)) info = broker.get_replication_info() diff --git a/swift/container/server.py b/swift/container/server.py index cfcdded1e4..9a6b4aa210 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -219,8 +219,6 @@ class ContainerController(object): if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) broker = self._get_container_broker(drive, part, account, container) - broker.pending_timeout = 0.1 - broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() @@ -246,8 +244,6 @@ class ContainerController(object): if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) broker = self._get_container_broker(drive, part, account, container) - broker.pending_timeout = 0.1 - broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() From a5d31b0d3a6206e14d3e55f37c43815fd309ad51 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Sun, 30 Jan 2011 08:59:35 -0600 Subject: [PATCH 18/64] removed extra import in account stats logger --- swift/stats/account_stats.py | 1 - 1 file changed, 1 deletion(-) diff --git a/swift/stats/account_stats.py b/swift/stats/account_stats.py index 6a9688831f..325746386c 100644 --- a/swift/stats/account_stats.py +++ b/swift/stats/account_stats.py @@ -21,7 +21,6 @@ import hashlib from swift.account.server import DATADIR as account_server_data_dir from swift.common.db import AccountBroker -from swift.common.internal_proxy import InternalProxy from swift.common.utils import renamer, get_logger, readconf, mkdirs from swift.common.constraints import check_mount from swift.common.daemon import Daemon From f457d772fbe3c9b9603eb809a5c1c4e6f7636514 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Tue, 1 Feb 2011 15:48:46 -0600 Subject: [PATCH 19/64] Bumping version to 1.2-rc --- swift/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/__init__.py b/swift/__init__.py index 316208f929..9e4b1ee0cd 100644 --- a/swift/__init__.py +++ b/swift/__init__.py @@ -1,5 +1,5 @@ import gettext -__version__ = '1.2-gamma' +__version__ = '1.2-rc' gettext.install('swift') From fdf20184e47383505865580560235af83ef9c35f Mon Sep 17 00:00:00 2001 From: gholt Date: Wed, 2 Feb 2011 09:38:17 -0800 Subject: [PATCH 20/64] Fix duplicate logging --- swift/common/utils.py | 1 + test/unit/common/test_daemon.py | 4 ++-- test/unit/common/test_utils.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index 479c4ecc30..a207153a74 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -409,6 +409,7 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None): logger = logging.getLogger() else: logger = logging.getLogger(log_route) + logger.propagate = False if not hasattr(get_logger, 'handlers'): get_logger.handlers = {} facility = getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'), diff --git a/test/unit/common/test_daemon.py b/test/unit/common/test_daemon.py index 015928f670..a4addcee51 100644 --- a/test/unit/common/test_daemon.py +++ b/test/unit/common/test_daemon.py @@ -28,7 +28,7 @@ class MyDaemon(daemon.Daemon): def __init__(self, conf): self.conf = conf - self.logger = utils.get_logger(None) + self.logger = utils.get_logger(None, 'server') MyDaemon.forever_called = False MyDaemon.once_called = False @@ -97,7 +97,7 @@ user = %s # test user quit MyDaemon.run_forever = MyDaemon.run_quit sio = StringIO() - logger = logging.getLogger() + logger = logging.getLogger('server') logger.addHandler(logging.StreamHandler(sio)) logger = utils.get_logger(None, 'server') daemon.run_daemon(MyDaemon, conf_file, logger=logger) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 94257415f1..0c81b15698 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -287,7 +287,7 @@ Error: unable to locate %s def test_get_logger(self): sio = StringIO() - logger = logging.getLogger() + logger = logging.getLogger('server') logger.addHandler(logging.StreamHandler(sio)) logger = utils.get_logger(None, 'server') logger.warn('test1') From 812fe86ea8108231039a5dbb06901f3de6bd71fb Mon Sep 17 00:00:00 2001 From: gholt Date: Wed, 2 Feb 2011 10:23:01 -0800 Subject: [PATCH 21/64] Make swauth only log requests it handles --- swift/common/middleware/swauth.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/swift/common/middleware/swauth.py b/swift/common/middleware/swauth.py index 3399fd06a4..5965e710ac 100644 --- a/swift/common/middleware/swauth.py +++ b/swift/common/middleware/swauth.py @@ -1321,6 +1321,8 @@ class Swauth(object): return False def posthooklogger(self, env, req): + if not req.path.startswith(self.auth_prefix): + return response = getattr(req, 'response', None) if not response: return From ee794fa79c95d3c9ee6eb8dd1ff0d00dd8165e8d Mon Sep 17 00:00:00 2001 From: gholt Date: Wed, 2 Feb 2011 10:47:56 -0800 Subject: [PATCH 22/64] logging: Remove old handler before installing a new handler --- swift/common/utils.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index a207153a74..8da3d1f8f4 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -410,28 +410,34 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None): else: logger = logging.getLogger(log_route) logger.propagate = False - if not hasattr(get_logger, 'handlers'): - get_logger.handlers = {} + if not hasattr(get_logger, 'handler4facility'): + get_logger.handler4facility = {} facility = getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'), SysLogHandler.LOG_LOCAL0) - if facility in get_logger.handlers: - logger.removeHandler(get_logger.handlers[facility]) - get_logger.handlers[facility].close() - del get_logger.handlers[facility] + if facility in get_logger.handler4facility: + logger.removeHandler(get_logger.handler4facility[facility]) + get_logger.handler4facility[facility].close() + del get_logger.handler4facility[facility] if log_to_console: # check if a previous call to get_logger already added a console logger if hasattr(get_logger, 'console') and get_logger.console: logger.removeHandler(get_logger.console) get_logger.console = logging.StreamHandler(sys.__stderr__) logger.addHandler(get_logger.console) - get_logger.handlers[facility] = \ + get_logger.handler4facility[facility] = \ SysLogHandler(address='/dev/log', facility=facility) - logger.addHandler(get_logger.handlers[facility]) + if not hasattr(get_logger, 'handler4logger'): + get_logger.handler4logger = {} + if logger in get_logger.handler4logger: + logger.removeHandler(get_logger.handler4logger[logger]) + get_logger.handler4logger[logger] = \ + get_logger.handler4facility[facility] + logger.addHandler(get_logger.handler4facility[facility]) logger.setLevel( getattr(logging, conf.get('log_level', 'INFO').upper(), logging.INFO)) adapted_logger = LogAdapter(logger) formatter = NamedFormatter(name, adapted_logger) - get_logger.handlers[facility].setFormatter(formatter) + get_logger.handler4facility[facility].setFormatter(formatter) if hasattr(get_logger, 'console'): get_logger.console.setFormatter(formatter) return adapted_logger From cb584303218805d1c1374d732caac26457a6ffe8 Mon Sep 17 00:00:00 2001 From: gholt Date: Wed, 2 Feb 2011 13:39:08 -0800 Subject: [PATCH 23/64] logging: use routes to separate logging configurations --- bin/swift-drive-audit | 2 +- bin/swift-log-uploader | 2 +- swift/account/auditor.py | 2 +- swift/account/reaper.py | 2 +- swift/account/server.py | 2 +- swift/auth/server.py | 2 +- swift/common/daemon.py | 4 ++-- swift/common/db_replicator.py | 2 +- swift/common/middleware/catch_errors.py | 2 +- swift/common/middleware/cname_lookup.py | 2 +- swift/common/middleware/ratelimit.py | 2 +- swift/common/middleware/swauth.py | 2 +- swift/common/utils.py | 2 ++ swift/common/wsgi.py | 2 +- swift/container/auditor.py | 2 +- swift/container/server.py | 2 +- swift/container/updater.py | 2 +- swift/obj/auditor.py | 2 +- swift/obj/replicator.py | 2 +- swift/obj/server.py | 2 +- swift/obj/updater.py | 2 +- swift/proxy/server.py | 2 +- swift/stats/access_processor.py | 2 +- swift/stats/account_stats.py | 3 ++- swift/stats/log_processor.py | 4 ++-- swift/stats/log_uploader.py | 3 ++- swift/stats/stats_processor.py | 2 +- test/unit/auth/test_server.py | 4 ++-- test/unit/common/test_daemon.py | 4 ++-- test/unit/common/test_utils.py | 8 +++++--- 30 files changed, 41 insertions(+), 35 deletions(-) diff --git a/bin/swift-drive-audit b/bin/swift-drive-audit index e92d1e3c12..5203f54b6b 100755 --- a/bin/swift-drive-audit +++ b/bin/swift-drive-audit @@ -99,7 +99,7 @@ if __name__ == '__main__': device_dir = conf.get('device_dir', '/srv/node') minutes = int(conf.get('minutes', 60)) error_limit = int(conf.get('error_limit', 1)) - logger = get_logger(conf, 'drive-audit') + logger = get_logger(conf, log_route='drive-audit') devices = get_devices(device_dir, logger) logger.debug("Devices found: %s" % str(devices)) if not devices: diff --git a/bin/swift-log-uploader b/bin/swift-log-uploader index 9d0e27836c..93cb8f6f97 100755 --- a/bin/swift-log-uploader +++ b/bin/swift-log-uploader @@ -34,7 +34,7 @@ if __name__ == '__main__': uploader_conf.update(plugin_conf) # pre-configure logger - logger = utils.get_logger(uploader_conf, plugin, + logger = utils.get_logger(uploader_conf, plugin, log_route='log-uploader', log_to_console=options.get('verbose', False)) # currently LogUploader only supports run_once options['once'] = True diff --git a/swift/account/auditor.py b/swift/account/auditor.py index 1f24f93acc..63551354d8 100644 --- a/swift/account/auditor.py +++ b/swift/account/auditor.py @@ -28,7 +28,7 @@ class AccountAuditor(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf, 'account-auditor') + self.logger = get_logger(conf, log_route='account-auditor') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/account/reaper.py b/swift/account/reaper.py index dd0d4b3890..ba78db8d98 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -53,7 +53,7 @@ class AccountReaper(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='account-reaper') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/account/server.py b/swift/account/server.py index 2c83f51cc6..f15ac38c11 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -42,7 +42,7 @@ class AccountController(object): """WSGI controller for the account server.""" def __init__(self, conf): - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='account-server') self.root = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/auth/server.py b/swift/auth/server.py index f9cd56dd0e..1258a706e7 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -90,7 +90,7 @@ class AuthController(object): """ def __init__(self, conf): - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='auth-server') self.super_admin_key = conf.get('super_admin_key') if not self.super_admin_key: msg = _('No super_admin_key set in conf file! Exiting.') diff --git a/swift/common/daemon.py b/swift/common/daemon.py index eee3428679..91230e4d2b 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -26,7 +26,7 @@ class Daemon(object): def __init__(self, conf): self.conf = conf - self.logger = utils.get_logger(conf, 'swift-daemon') + self.logger = utils.get_logger(conf, log_route='daemon') def run_once(self): """Override this to run the script once""" @@ -84,7 +84,7 @@ def run_daemon(klass, conf_file, section_name='', logger = kwargs.pop('logger') else: logger = utils.get_logger(conf, conf.get('log_name', section_name), - log_to_console=kwargs.pop('verbose', False)) + log_to_console=kwargs.pop('verbose', False), log_route=section_name) try: klass(conf).run(once=once, **kwargs) except KeyboardInterrupt: diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 49756f1f7b..3c3731d45a 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -92,7 +92,7 @@ class Replicator(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='replicator') self.root = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/common/middleware/catch_errors.py b/swift/common/middleware/catch_errors.py index 10d8614194..16ade84689 100644 --- a/swift/common/middleware/catch_errors.py +++ b/swift/common/middleware/catch_errors.py @@ -30,7 +30,7 @@ class CatchErrorMiddleware(object): self.logger = getattr(app, 'logger', None) if not self.logger: # and only call get_logger if we have to - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='catch-errors') def __call__(self, env, start_response): try: diff --git a/swift/common/middleware/cname_lookup.py b/swift/common/middleware/cname_lookup.py index f13155c1fe..8ea9f88071 100644 --- a/swift/common/middleware/cname_lookup.py +++ b/swift/common/middleware/cname_lookup.py @@ -53,7 +53,7 @@ class CNAMELookupMiddleware(object): self.storage_domain = '.' + self.storage_domain self.lookup_depth = int(conf.get('lookup_depth', '1')) self.memcache = None - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='cname-lookup') def __call__(self, env, start_response): if not self.storage_domain: diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index 4657b6abcd..485b1db26e 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -39,7 +39,7 @@ class RateLimitMiddleware(object): if logger: self.logger = logger else: - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='ratelimit') self.account_ratelimit = float(conf.get('account_ratelimit', 0)) self.max_sleep_time_seconds = \ float(conf.get('max_sleep_time_seconds', 60)) diff --git a/swift/common/middleware/swauth.py b/swift/common/middleware/swauth.py index 5965e710ac..32328e8eb6 100644 --- a/swift/common/middleware/swauth.py +++ b/swift/common/middleware/swauth.py @@ -51,7 +51,7 @@ class Swauth(object): def __init__(self, app, conf): self.app = app self.conf = conf - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='swauth') self.log_headers = conf.get('log_headers') == 'True' self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() if self.reseller_prefix and self.reseller_prefix[-1] != '_': diff --git a/swift/common/utils.py b/swift/common/utils.py index 8da3d1f8f4..698fae2cc1 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -395,6 +395,8 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None): :param conf: Configuration dict to read settings from :param name: Name of the logger :param log_to_console: Add handler which writes to console on stderr + :param log_route: Route for the logging, not emitted to the log, just used + to separate logging configurations """ if not conf: conf = {} diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 9450bcf439..46207fe8e1 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -113,7 +113,7 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): logger = kwargs.pop('logger') else: logger = get_logger(conf, log_name, - log_to_console=kwargs.pop('verbose', False)) + log_to_console=kwargs.pop('verbose', False), log_route='wsgi') # redirect errors to logger and close stdio capture_stdio(logger) diff --git a/swift/container/auditor.py b/swift/container/auditor.py index d1ceb4f98a..0b1c10e03e 100644 --- a/swift/container/auditor.py +++ b/swift/container/auditor.py @@ -28,7 +28,7 @@ class ContainerAuditor(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf, 'container-auditor') + self.logger = get_logger(conf, log_route='container-auditor') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/container/server.py b/swift/container/server.py index cfcdded1e4..561dad2ea9 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -49,7 +49,7 @@ class ContainerController(object): save_headers = ['x-container-read', 'x-container-write'] def __init__(self, conf): - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='container-server') self.root = conf.get('devices', '/srv/node/') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/container/updater.py b/swift/container/updater.py index 883dd17101..0bd000f3f2 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -37,7 +37,7 @@ class ContainerUpdater(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf, 'container-updater') + self.logger = get_logger(conf, log_route='container-updater') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 09fdd77774..8ed05049f3 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -31,7 +31,7 @@ class ObjectAuditor(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf, 'object-auditor') + self.logger = get_logger(conf, log_route='object-auditor') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index dcfcb926f9..8dec8aa801 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -207,7 +207,7 @@ class ObjectReplicator(Daemon): :param logger: logging object """ self.conf = conf - self.logger = get_logger(conf, 'object-replicator') + self.logger = get_logger(conf, log_route='object-replicator') self.devices_dir = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/obj/server.py b/swift/obj/server.py index f2e2b31314..e3626bf692 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -266,7 +266,7 @@ class ObjectController(object): /etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='object-server') self.devices = conf.get('devices', '/srv/node/') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/obj/updater.py b/swift/obj/updater.py index 2b28ff08c5..356be64da4 100644 --- a/swift/obj/updater.py +++ b/swift/obj/updater.py @@ -35,7 +35,7 @@ class ObjectUpdater(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf, 'object-updater') + self.logger = get_logger(conf, log_route='object-updater') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 1eae0dfc30..a66f643a68 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -1607,7 +1607,7 @@ class BaseApplication(object): def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, object_ring=None): if logger is None: - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='proxy-server') else: self.logger = logger if conf is None: diff --git a/swift/stats/access_processor.py b/swift/stats/access_processor.py index 2aee505415..6965ef2b4a 100644 --- a/swift/stats/access_processor.py +++ b/swift/stats/access_processor.py @@ -34,7 +34,7 @@ class AccessLogProcessor(object): conf.get('service_ips', '').split(',')\ if x.strip()] self.warn_percent = float(conf.get('warn_percent', '0.8')) - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='access-processor') def log_line_parser(self, raw_log): '''given a raw access log line, return a dict of the good parts''' diff --git a/swift/stats/account_stats.py b/swift/stats/account_stats.py index 325746386c..34b024d2c2 100644 --- a/swift/stats/account_stats.py +++ b/swift/stats/account_stats.py @@ -48,7 +48,8 @@ class AccountStat(Daemon): self.devices = server_conf.get('devices', '/srv/node') self.mount_check = server_conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.logger = get_logger(stats_conf, 'swift-account-stats-logger') + self.logger = \ + get_logger(stats_conf, log_route='account-stats') def run_once(self): self.logger.info(_("Gathering account stats")) diff --git a/swift/stats/log_processor.py b/swift/stats/log_processor.py index 5dbc92afbe..727e687f38 100644 --- a/swift/stats/log_processor.py +++ b/swift/stats/log_processor.py @@ -40,7 +40,7 @@ class LogProcessor(object): def __init__(self, conf, logger): if isinstance(logger, tuple): - self.logger = get_logger(*logger) + self.logger = get_logger(*logger, log_route='log-processor') else: self.logger = logger @@ -226,7 +226,7 @@ class LogProcessorDaemon(Daemon): c = conf.get('log-processor') super(LogProcessorDaemon, self).__init__(c) self.total_conf = conf - self.logger = get_logger(c) + self.logger = get_logger(c, log_route='log-processor') self.log_processor = LogProcessor(conf, self.logger) self.lookback_hours = int(c.get('lookback_hours', '120')) self.lookback_window = int(c.get('lookback_window', diff --git a/swift/stats/log_uploader.py b/swift/stats/log_uploader.py index b425738938..a828188eb7 100644 --- a/swift/stats/log_uploader.py +++ b/swift/stats/log_uploader.py @@ -65,7 +65,8 @@ class LogUploader(Daemon): self.filename_format = source_filename_format self.internal_proxy = InternalProxy(proxy_server_conf) log_name = 'swift-log-uploader-%s' % plugin_name - self.logger = utils.get_logger(uploader_conf, plugin_name) + self.logger = \ + utils.get_logger(uploader_conf, plugin_name, log_route=plugin_name) def run_once(self): self.logger.info(_("Uploading logs")) diff --git a/swift/stats/stats_processor.py b/swift/stats/stats_processor.py index 95dba7604c..f9496c1df9 100644 --- a/swift/stats/stats_processor.py +++ b/swift/stats/stats_processor.py @@ -20,7 +20,7 @@ class StatsLogProcessor(object): """Transform account storage stat logs""" def __init__(self, conf): - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='stats-processor') def process(self, obj_stream, data_object_account, data_object_container, data_object_name): diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py index 4060766d65..d58556ab22 100644 --- a/test/unit/auth/test_server.py +++ b/test/unit/auth/test_server.py @@ -456,7 +456,7 @@ class TestAuthServer(unittest.TestCase): def test_basic_logging(self): log = StringIO() log_handler = StreamHandler(log) - logger = get_logger(self.conf, 'auth') + logger = get_logger(self.conf, 'auth-server', log_route='auth-server') logger.logger.addHandler(log_handler) try: auth_server.http_connect = fake_http_connect(201) @@ -534,7 +534,7 @@ class TestAuthServer(unittest.TestCase): orig_Request = auth_server.Request log = StringIO() log_handler = StreamHandler(log) - logger = get_logger(self.conf, 'auth') + logger = get_logger(self.conf, 'auth-server', log_route='auth-server') logger.logger.addHandler(log_handler) try: auth_server.Request = request_causing_exception diff --git a/test/unit/common/test_daemon.py b/test/unit/common/test_daemon.py index a4addcee51..1d54e78c3e 100644 --- a/test/unit/common/test_daemon.py +++ b/test/unit/common/test_daemon.py @@ -28,7 +28,7 @@ class MyDaemon(daemon.Daemon): def __init__(self, conf): self.conf = conf - self.logger = utils.get_logger(None, 'server') + self.logger = utils.get_logger(None, 'server', log_route='server') MyDaemon.forever_called = False MyDaemon.once_called = False @@ -99,7 +99,7 @@ user = %s sio = StringIO() logger = logging.getLogger('server') logger.addHandler(logging.StreamHandler(sio)) - logger = utils.get_logger(None, 'server') + logger = utils.get_logger(None, 'server', log_route='server') daemon.run_daemon(MyDaemon, conf_file, logger=logger) self.assert_('user quit' in sio.getvalue().lower()) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 0c81b15698..8da913c489 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -289,17 +289,19 @@ Error: unable to locate %s sio = StringIO() logger = logging.getLogger('server') logger.addHandler(logging.StreamHandler(sio)) - logger = utils.get_logger(None, 'server') + logger = utils.get_logger(None, 'server', log_route='server') logger.warn('test1') self.assertEquals(sio.getvalue(), 'test1\n') logger.debug('test2') self.assertEquals(sio.getvalue(), 'test1\n') - logger = utils.get_logger({'log_level': 'DEBUG'}, 'server') + logger = utils.get_logger({'log_level': 'DEBUG'}, 'server', + log_route='server') logger.debug('test3') self.assertEquals(sio.getvalue(), 'test1\ntest3\n') # Doesn't really test that the log facility is truly being used all the # way to syslog; but exercises the code. - logger = utils.get_logger({'log_facility': 'LOG_LOCAL3'}, 'server') + logger = utils.get_logger({'log_facility': 'LOG_LOCAL3'}, 'server', + log_route='server') logger.warn('test4') self.assertEquals(sio.getvalue(), 'test1\ntest3\ntest4\n') From bf34239d79cc78cafa138f02fcde327ea4db69c1 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Wed, 2 Feb 2011 22:37:55 +0000 Subject: [PATCH 24/64] Bumping version to 1.2.0 in preparation for release --- swift/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/__init__.py b/swift/__init__.py index 9e4b1ee0cd..899047889e 100644 --- a/swift/__init__.py +++ b/swift/__init__.py @@ -1,5 +1,5 @@ import gettext -__version__ = '1.2-rc' +__version__ = '1.2.0' gettext.install('swift') From 4bee91515e00bc32b487df767c629f62d77aec8e Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Thu, 3 Feb 2011 19:27:25 +0000 Subject: [PATCH 25/64] Updating version to 1.3-dev --- swift/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/__init__.py b/swift/__init__.py index 899047889e..25a1c6b8c7 100644 --- a/swift/__init__.py +++ b/swift/__init__.py @@ -1,5 +1,5 @@ import gettext -__version__ = '1.2.0' +__version__ = '1.3-dev' gettext.install('swift') From ee3886f2ca45e8983b13af04a06fee000aab61bb Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 3 Feb 2011 13:46:28 -0600 Subject: [PATCH 26/64] moved warning messages out of proxy.logger.info A few warning/client error messages were useing the .info log level which we reserve for access logs. They were changed to warnings. --- swift/proxy/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 1eae0dfc30..14f79a2c61 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -624,7 +624,7 @@ class Controller(object): res.bytes_transferred += len(chunk) except GeneratorExit: res.client_disconnect = True - self.app.logger.info(_('Client disconnected on read')) + self.app.logger.warn(_('Client disconnected on read')) except (Exception, TimeoutError): self.exception_occurred(node, _('Object'), _('Trying to read during GET of %s') % req.path) @@ -1054,7 +1054,7 @@ class ObjectController(Controller): if req.headers.get('transfer-encoding') and chunk == '': break except ChunkReadTimeout, err: - self.app.logger.info( + self.app.logger.warn( _('ERROR Client read timeout (%ss)'), err.seconds) return HTTPRequestTimeout(request=req) except Exception: @@ -1064,7 +1064,7 @@ class ObjectController(Controller): return Response(status='499 Client Disconnect') if req.content_length and req.bytes_transferred < req.content_length: req.client_disconnect = True - self.app.logger.info( + self.app.logger.warn( _('Client disconnected without sending enough data')) return Response(status='499 Client Disconnect') statuses = [] From c62842bfd111c9f2ac4dbd343582332452ec13b3 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Thu, 3 Feb 2011 19:50:16 +0000 Subject: [PATCH 27/64] update all ring-builder grammars --- bin/swift-ring-builder | 47 +++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/bin/swift-ring-builder b/bin/swift-ring-builder index 41293f7d37..c6d91f92b4 100755 --- a/bin/swift-ring-builder +++ b/bin/swift-ring-builder @@ -48,6 +48,8 @@ The can be of the form: /sdb1 Matches devices with the device name sdb1 _shiny Matches devices with shiny in the meta data _"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data + [::1] Matches devices in any zone with the ip ::1 + z1-[::1]:5678 Matches devices in zone 1 with the ip ::1 and port 5678 Most specific example: d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8" Nerd explanation: @@ -76,6 +78,13 @@ The can be of the form: i += 1 match.append(('ip', search_value[:i])) search_value = search_value[i:] + elif len(search_value) and search_value[0] == '[': + i = 1 + while i < len(search_value) and search_value[i] != ']': + i += 1 + i += 1 + match.append(('ip', search_value[:i].lstrip('[').rstrip(']'))) + search_value = search_value[i:] if search_value.startswith(':'): i = 1 while i < len(search_value) and search_value[i].isdigit(): @@ -110,6 +119,16 @@ The can be of the form: return devs +def format_device(dev): + """ + Format a device for display. + """ + if ':' in dev['ip']: + return 'd%(id)sz%(zone)s-[%(ip)s]:%(port)s/%(device)s_"%(meta)s"' % dev + else: + return 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev + + class Commands: def unknown(): @@ -236,10 +255,11 @@ swift-ring-builder add z-:/_ exit(EXIT_ERROR) i = 1 if rest[i] == '[': + i += 1 while i < len(rest) and rest[i] != ']': i += 1 - ip = rest[2:i] i += 1 + ip = rest[1:i].lstrip('[').rstrip(']') rest = rest[i:] else: while i < len(rest) and rest[i] in '0123456789.': @@ -286,8 +306,12 @@ swift-ring-builder add z-:/_ builder.add_dev({'id': next_dev_id, 'zone': zone, 'ip': ip, 'port': port, 'device': device_name, 'weight': weight, 'meta': meta}) - print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \ - (zone, ip, port, device_name, meta, weight, next_dev_id) + if ':' in ip: + print 'Device z%s-[%s]:%s/%s_"%s" with %s weight got id %s' % \ + (zone, ip, port, device_name, meta, weight, next_dev_id) + else: + print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \ + (zone, ip, port, device_name, meta, weight, next_dev_id) pickle.dump(builder, open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) @@ -349,6 +373,13 @@ swift-ring-builder set_info i += 1 change.append(('ip', change_value[:i])) change_value = change_value[i:] + elif len(change_value) and change_value[0] == '[': + i = 1 + while i < len(change_value) and change_value[i] != ']': + i += 1 + i += 1 + change.append(('ip', change_value[:i].lstrip('[').rstrip(']'))) + change_value = change_value[i:] if change_value.startswith(':'): i = 1 while i < len(change_value) and change_value[i].isdigit(): @@ -373,15 +404,13 @@ swift-ring-builder set_info if len(devs) > 1: print 'Matched more than one device:' for dev in devs: - print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ - '"%(meta)s"' % dev + print ' %s' % format_device(dev) if raw_input('Are you sure you want to update the info for ' 'these %s devices? (y/N) ' % len(devs)) != 'y': print 'Aborting device modifications' exit(EXIT_ERROR) for dev in devs: - orig_dev_string = \ - 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev + orig_dev_string = format_device(dev) test_dev = dict(dev) for key, value in change: test_dev[key] = value @@ -397,9 +426,7 @@ swift-ring-builder set_info exit(EXIT_ERROR) for key, value in change: dev[key] = value - new_dev_string = \ - 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev - print 'Device %s is now %s' % (orig_dev_string, new_dev_string) + print 'Device %s is now %s' % (orig_dev_string, format_device(dev)) pickle.dump(builder, open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) From c2931e157c4db364e4ce9d971920be2a83641431 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Thu, 3 Feb 2011 19:53:47 +0000 Subject: [PATCH 28/64] random newline --- test/unit/common/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 69d24759b7..af38b18208 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -501,5 +501,6 @@ log_name = yarr''' # make sure its accurate to 10th of a second self.assertTrue(abs(100 - (time.time() - start) * 100) < 10) + if __name__ == '__main__': unittest.main() From f9fa63686c802ce8d3f2e4e29ecc7fb686835ba9 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 3 Feb 2011 15:23:07 -0600 Subject: [PATCH 29/64] Moved proxy server access log messages into their own log level Added new "access" log level available on swift loggers that will be routed to the LOG_NOTICE priority in syslog for easy redirection of access log messages via rsyslog and syslog-ng. --- doc/source/overview_stats.rst | 15 ++++++++------- swift/common/utils.py | 15 +++++++++++++++ swift/proxy/server.py | 2 +- test/unit/common/test_utils.py | 6 ++++++ test/unit/proxy/test_server.py | 2 +- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/doc/source/overview_stats.rst b/doc/source/overview_stats.rst index 6364de4611..40a5dd01af 100644 --- a/doc/source/overview_stats.rst +++ b/doc/source/overview_stats.rst @@ -15,9 +15,10 @@ Access logs *********** Access logs are the proxy server logs. Rackspace uses syslog-ng to redirect -the proxy log output to an hourly log file. For example, a proxy request that -is made on August 4, 2010 at 12:37 gets logged in a file named 2010080412. -This allows easy log rotation and easy per-hour log processing. +proxy log messages with the syslog priority LOG_NOTICE to an hourly log +file. For example, a proxy request that is made on August 4, 2010 at 12:37 gets +logged in a file named 2010080412. This allows easy log rotation and easy +per-hour log processing. ****************** Account stats logs @@ -99,11 +100,11 @@ Running the stats system on SAIO destination df_local1 { file("/var/log/swift/proxy.log" owner() group()); }; destination df_local1_err { file("/var/log/swift/proxy.error" owner() group()); }; destination df_local1_hourly { file("/var/log/swift/hourly/$YEAR$MONTH$DAY$HOUR" owner() group()); }; - filter f_local1 { facility(local1) and level(info); }; + filter f_local1 { facility(local1) and level(notice); }; - filter f_local1_err { facility(local1) and not level(info); }; + filter f_local1_err { facility(local1) and not level(notice); }; - # local1.info -/var/log/swift/proxy.log + # local1.notice -/var/log/swift/proxy.log # write to local file and to remove log server log { source(s_all); @@ -181,4 +182,4 @@ earlier. This file will have one entry per account per hour for each account with activity in that hour. One .csv file should be produced per hour. Note that the stats will be delayed by at least two hours by default. This can be changed with the new_log_cutoff variable in the config file. See -`log-processing.conf-sample` for more details. \ No newline at end of file +`log-processing.conf-sample` for more details. diff --git a/swift/common/utils.py b/swift/common/utils.py index 8da3d1f8f4..5c462b5d08 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -48,6 +48,11 @@ import logging logging.thread = eventlet.green.thread logging.threading = eventlet.green.threading logging._lock = logging.threading.RLock() +# setup access level logging +ACCESS = 25 +logging._levelNames[ACCESS] = 'ACCESS' +# syslog priority "notice" is used for proxy access log lines +SysLogHandler.priority_map['ACCESS'] = 'notice' # These are lazily pulled from libc elsewhere _sys_fallocate = None @@ -310,6 +315,16 @@ class LogAdapter(object): def getEffectiveLevel(self): return self.logger.getEffectiveLevel() + def access(self, msg, *args): + """ + Convenience function for proxy access request log level. Only + proxy access log messages should use this method. The python + logging lvl is set to 25, just above info. SysLogHandler is + monkey patched to map this log lvl to the LOG_NOTICE syslog + priority. + """ + self.logger.log(ACCESS, msg, *args) + def exception(self, msg, *args): _junk, exc, _junk = sys.exc_info() call = self.logger.error diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 14f79a2c61..dc501faba5 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -1790,7 +1790,7 @@ class Application(BaseApplication): if getattr(req, 'client_disconnect', False) or \ getattr(response, 'client_disconnect', False): status_int = 499 - self.logger.info(' '.join(quote(str(x)) for x in ( + self.logger.access(' '.join(quote(str(x)) for x in ( client or '-', req.remote_addr or '-', time.strftime('%d/%b/%Y/%H/%M/%S', time.gmtime()), diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 0c81b15698..1fd18b6ee6 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -303,9 +303,15 @@ Error: unable to locate %s logger.warn('test4') self.assertEquals(sio.getvalue(), 'test1\ntest3\ntest4\n') + # make sure debug doesn't log by default logger.debug('test5') self.assertEquals(sio.getvalue(), 'test1\ntest3\ntest4\n') + # make sure access lvl logs by default + logger.access('test6') + self.assertEquals(sio.getvalue(), + 'test1\ntest3\ntest4\ntest6\n') + def test_storage_directory(self): self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'), diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index e991d84084..4b0404454f 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1802,7 +1802,7 @@ class TestObjectController(unittest.TestCase): class Logger(object): - def info(self, msg): + def access(self, msg): self.msg = msg orig_logger = prosrv.logger From ee4a9a85ac8763b14deb9c55e6c9be2a163bb5a8 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 4 Feb 2011 11:16:21 -0800 Subject: [PATCH 30/64] Indexing and integrity changes in dbs. --- swift/common/db.py | 125 +++++++++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 56 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index 7040b2446a..4327ffa311 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -166,6 +166,7 @@ class DatabaseBroker(object): self.logger = logger or logging.getLogger() self.account = account self.container = container + self.db_version = -1 def initialize(self, put_timestamp=None): """ @@ -573,7 +574,7 @@ class ContainerBroker(DatabaseBroker): conn.executescript(""" CREATE TABLE object ( ROWID INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT UNIQUE, + name TEXT, created_at TEXT, size INTEGER, content_type TEXT, @@ -581,7 +582,7 @@ class ContainerBroker(DatabaseBroker): deleted INTEGER DEFAULT 0 ); - CREATE INDEX ix_object_deleted ON object (deleted); + CREATE INDEX ix_object_deleted_name ON object (deleted, name); CREATE TRIGGER object_insert AFTER INSERT ON object BEGIN @@ -812,6 +813,12 @@ class ContainerBroker(DatabaseBroker): reported_object_count, reported_bytes_used, hash, id) """ with self.get() as conn: + if self.db_version == -1: + self.db_version = 0 + for row in conn.execute(''' + SELECT name FROM sqlite_master + WHERE name = 'ix_object_deleted_name' '''): + self.db_version = 1 return conn.execute(''' SELECT account, container, created_at, put_timestamp, delete_timestamp, object_count, bytes_used, @@ -906,7 +913,10 @@ class ContainerBroker(DatabaseBroker): elif prefix: query += ' name >= ? AND' query_args.append(prefix) - query += ' +deleted = 0 ORDER BY name LIMIT ?' + if self.db_version < 1: + query += ' +deleted = 0 ORDER BY name LIMIT ?' + else: + query += ' deleted = 0 ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None @@ -954,18 +964,19 @@ class ContainerBroker(DatabaseBroker): max_rowid = -1 for rec in item_list: conn.execute(''' - DELETE FROM object WHERE name = ? AND - (created_at < ?) + DELETE FROM object WHERE name = ? AND created_at < ? AND + deleted IN (0, 1) ''', (rec['name'], rec['created_at'])) - try: + if not conn.execute(''' + SELECT name FROM object WHERE name = ? AND + deleted IN (0, 1) + ''', (rec['name'],)).fetchall(): conn.execute(''' INSERT INTO object (name, created_at, size, content_type, etag, deleted) VALUES (?, ?, ?, ?, ?, ?) ''', ([rec['name'], rec['created_at'], rec['size'], rec['content_type'], rec['etag'], rec['deleted']])) - except sqlite3.IntegrityError: - pass if source: max_rowid = max(max_rowid, rec['ROWID']) if source: @@ -1009,7 +1020,7 @@ class AccountBroker(DatabaseBroker): conn.executescript(""" CREATE TABLE container ( ROWID INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT UNIQUE, + name TEXT, put_timestamp TEXT, delete_timestamp TEXT, object_count INTEGER, @@ -1017,8 +1028,9 @@ class AccountBroker(DatabaseBroker): deleted INTEGER DEFAULT 0 ); - CREATE INDEX ix_container_deleted ON container (deleted); - CREATE INDEX ix_container_name ON container (name); + CREATE INDEX ix_container_deleted_name ON + container (deleted, name); + CREATE TRIGGER container_insert AFTER INSERT ON container BEGIN UPDATE account_stat @@ -1287,6 +1299,12 @@ class AccountBroker(DatabaseBroker): bytes_used, hash, id) """ with self.get() as conn: + if self.db_version == -1: + self.db_version = 0 + for row in conn.execute(''' + SELECT name FROM sqlite_master + WHERE name = 'ix_container_deleted_name' '''): + self.db_version = 1 return conn.execute(''' SELECT account, created_at, put_timestamp, delete_timestamp, container_count, object_count, bytes_used, hash, id @@ -1355,7 +1373,10 @@ class AccountBroker(DatabaseBroker): elif prefix: query += ' name >= ? AND' query_args.append(prefix) - query += ' +deleted = 0 ORDER BY name LIMIT ?' + if self.db_version < 1: + query += ' +deleted = 0 ORDER BY name LIMIT ?' + else: + query += ' deleted = 0 ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None @@ -1399,51 +1420,43 @@ class AccountBroker(DatabaseBroker): record = [rec['name'], rec['put_timestamp'], rec['delete_timestamp'], rec['object_count'], rec['bytes_used'], rec['deleted']] - try: + curs = conn.execute(''' + SELECT name, put_timestamp, delete_timestamp, + object_count, bytes_used, deleted + FROM container WHERE name = ? AND + (put_timestamp < ? OR delete_timestamp < ? OR + object_count != ? OR bytes_used != ?) AND + deleted IN (0, 1)''', + (rec['name'], rec['put_timestamp'], + rec['delete_timestamp'], rec['object_count'], + rec['bytes_used'])) + curs.row_factory = None + row = curs.fetchone() + if row: + row = list(row) + for i in xrange(5): + if record[i] is None and row[i] is not None: + record[i] = row[i] + if row[1] > record[1]: # Keep newest put_timestamp + record[1] = row[1] + if row[2] > record[2]: # Keep newest delete_timestamp + record[2] = row[2] conn.execute(''' - INSERT INTO container (name, put_timestamp, - delete_timestamp, object_count, bytes_used, - deleted) - VALUES (?, ?, ?, ?, ?, ?) - ''', record) - except sqlite3.IntegrityError: - curs = conn.execute(''' - SELECT name, put_timestamp, delete_timestamp, - object_count, bytes_used, deleted - FROM container WHERE name = ? AND - (put_timestamp < ? OR delete_timestamp < ? OR - object_count != ? OR bytes_used != ?)''', - (rec['name'], rec['put_timestamp'], - rec['delete_timestamp'], rec['object_count'], - rec['bytes_used'])) - curs.row_factory = None - row = curs.fetchone() - if row: - row = list(row) - for i in xrange(5): - if record[i] is None and row[i] is not None: - record[i] = row[i] - if row[1] > record[1]: # Keep newest put_timestamp - record[1] = row[1] - if row[2] > record[2]: # Keep newest delete_timestamp - record[2] = row[2] - conn.execute('DELETE FROM container WHERE name = ?', - (record[0],)) - # If deleted, mark as such - if record[2] > record[1] and \ - record[3] in (None, '', 0, '0'): - record[5] = 1 - else: - record[5] = 0 - try: - conn.execute(''' - INSERT INTO container (name, put_timestamp, - delete_timestamp, object_count, bytes_used, - deleted) - VALUES (?, ?, ?, ?, ?, ?) - ''', record) - except sqlite3.IntegrityError: - continue + DELETE FROM container WHERE name = ? AND + deleted IN (0, 1) + ''', (record[0],)) + # If deleted, mark as such + if record[2] > record[1] and \ + record[3] in (None, '', 0, '0'): + record[5] = 1 + else: + record[5] = 0 + conn.execute(''' + INSERT INTO container (name, put_timestamp, + delete_timestamp, object_count, bytes_used, + deleted) + VALUES (?, ?, ?, ?, ?, ?) + ''', record) if source: max_rowid = max(max_rowid, rec['ROWID']) if source: From 2fffdfede24f79df757faffd197afff32ca432eb Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 4 Feb 2011 11:37:35 -0800 Subject: [PATCH 31/64] Move db version resolution to its own function --- swift/common/db.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index 4327ffa311..2341f8141f 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -166,7 +166,7 @@ class DatabaseBroker(object): self.logger = logger or logging.getLogger() self.account = account self.container = container - self.db_version = -1 + self._db_version = -1 def initialize(self, put_timestamp=None): """ @@ -645,6 +645,15 @@ class ContainerBroker(DatabaseBroker): ''', (self.account, self.container, normalize_timestamp(time.time()), str(uuid4()), put_timestamp)) + def _get_db_version(self, conn): + if self._db_version == -1: + self._db_version = 0 + for row in conn.execute(''' + SELECT name FROM sqlite_master + WHERE name = 'ix_object_deleted_name' '''): + self._db_version = 1 + return self._db_version + def _newid(self, conn): conn.execute(''' UPDATE container_stat @@ -813,12 +822,6 @@ class ContainerBroker(DatabaseBroker): reported_object_count, reported_bytes_used, hash, id) """ with self.get() as conn: - if self.db_version == -1: - self.db_version = 0 - for row in conn.execute(''' - SELECT name FROM sqlite_master - WHERE name = 'ix_object_deleted_name' '''): - self.db_version = 1 return conn.execute(''' SELECT account, container, created_at, put_timestamp, delete_timestamp, object_count, bytes_used, @@ -913,7 +916,7 @@ class ContainerBroker(DatabaseBroker): elif prefix: query += ' name >= ? AND' query_args.append(prefix) - if self.db_version < 1: + if self._get_db_version(conn) < 1: query += ' +deleted = 0 ORDER BY name LIMIT ?' else: query += ' deleted = 0 ORDER BY name LIMIT ?' @@ -1094,6 +1097,15 @@ class AccountBroker(DatabaseBroker): ''', (self.account, normalize_timestamp(time.time()), str(uuid4()), put_timestamp)) + def _get_db_version(self, conn): + if self._db_version == -1: + self._db_version = 0 + for row in conn.execute(''' + SELECT name FROM sqlite_master + WHERE name = 'ix_container_deleted_name' '''): + self._db_version = 1 + return self._db_version + def update_put_timestamp(self, timestamp): """ Update the put_timestamp. Only modifies it if it is greater than @@ -1299,12 +1311,6 @@ class AccountBroker(DatabaseBroker): bytes_used, hash, id) """ with self.get() as conn: - if self.db_version == -1: - self.db_version = 0 - for row in conn.execute(''' - SELECT name FROM sqlite_master - WHERE name = 'ix_container_deleted_name' '''): - self.db_version = 1 return conn.execute(''' SELECT account, created_at, put_timestamp, delete_timestamp, container_count, object_count, bytes_used, hash, id @@ -1373,7 +1379,7 @@ class AccountBroker(DatabaseBroker): elif prefix: query += ' name >= ? AND' query_args.append(prefix) - if self.db_version < 1: + if self._get_db_version(conn) < 1: query += ' +deleted = 0 ORDER BY name LIMIT ?' else: query += ' deleted = 0 ORDER BY name LIMIT ?' From 98090b7217c69bba06b2f9ecb1dfaceb29de877d Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 4 Feb 2011 11:50:30 -0800 Subject: [PATCH 32/64] Fix account db change --- swift/common/db.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index 2341f8141f..83cd0e8188 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -1430,12 +1430,8 @@ class AccountBroker(DatabaseBroker): SELECT name, put_timestamp, delete_timestamp, object_count, bytes_used, deleted FROM container WHERE name = ? AND - (put_timestamp < ? OR delete_timestamp < ? OR - object_count != ? OR bytes_used != ?) AND - deleted IN (0, 1)''', - (rec['name'], rec['put_timestamp'], - rec['delete_timestamp'], rec['object_count'], - rec['bytes_used'])) + deleted IN (0, 1) + ''', (rec['name'],)) curs.row_factory = None row = curs.fetchone() if row: @@ -1447,16 +1443,16 @@ class AccountBroker(DatabaseBroker): record[1] = row[1] if row[2] > record[2]: # Keep newest delete_timestamp record[2] = row[2] - conn.execute(''' - DELETE FROM container WHERE name = ? AND - deleted IN (0, 1) - ''', (record[0],)) # If deleted, mark as such if record[2] > record[1] and \ record[3] in (None, '', 0, '0'): record[5] = 1 else: record[5] = 0 + conn.execute(''' + DELETE FROM container WHERE name = ? AND + deleted IN (0, 1) + ''', (record[0],)) conn.execute(''' INSERT INTO container (name, put_timestamp, delete_timestamp, object_count, bytes_used, From 0f0e093972dccad7ec00f7e3fd73573aa09f4f46 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Fri, 4 Feb 2011 20:58:31 -0600 Subject: [PATCH 33/64] fix st command help hangs Before running the command function global the main func would start the print and error queues. Inside the command function the the option parser would see the the help option, print the help text, and raise SystemExit, which wasn't getting caught. --- bin/st | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/st b/bin/st index 58285423bd..4e6024f84f 100755 --- a/bin/st +++ b/bin/st @@ -1723,7 +1723,7 @@ Example: error_thread.abort = True while error_thread.isAlive(): error_thread.join(0.01) - except Exception: + except (SystemExit, Exception): for thread in threading_enumerate(): thread.abort = True raise From 461bf8df712f1b03ed547bdf0a068aae434d50ef Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Sat, 5 Feb 2011 15:38:49 -0600 Subject: [PATCH 34/64] added new proxy-server configuration options for access_log_facility and access_log_name --- swift/common/utils.py | 15 +++++++-------- swift/common/wsgi.py | 8 ++++---- swift/proxy/server.py | 12 +++++++++++- test/unit/common/test_utils.py | 4 ++-- test/unit/proxy/test_server.py | 10 +++++++--- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index 5c462b5d08..595ad3ac03 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -48,11 +48,11 @@ import logging logging.thread = eventlet.green.thread logging.threading = eventlet.green.threading logging._lock = logging.threading.RLock() -# setup access level logging -ACCESS = 25 -logging._levelNames[ACCESS] = 'ACCESS' +# setup notice level logging +NOTICE = 25 +logging._levelNames[NOTICE] = 'NOTICE' # syslog priority "notice" is used for proxy access log lines -SysLogHandler.priority_map['ACCESS'] = 'notice' +SysLogHandler.priority_map['NOTICE'] = 'notice' # These are lazily pulled from libc elsewhere _sys_fallocate = None @@ -315,15 +315,14 @@ class LogAdapter(object): def getEffectiveLevel(self): return self.logger.getEffectiveLevel() - def access(self, msg, *args): + def notice(self, msg, *args): """ - Convenience function for proxy access request log level. Only - proxy access log messages should use this method. The python + Convenience function for syslog priority LOG_NOTICE. The python logging lvl is set to 25, just above info. SysLogHandler is monkey patched to map this log lvl to the LOG_NOTICE syslog priority. """ - self.logger.log(ACCESS, msg, *args) + self.logger.log(NOTICE, msg, *args) def exception(self, msg, *args): _junk, exc, _junk = sys.exc_info() diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 9450bcf439..e1e6e0c8f1 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -168,10 +168,10 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): signal.signal(signal.SIGHUP, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) run_server() - logger.info('Child %d exiting normally' % os.getpid()) + logger.notice('Child %d exiting normally' % os.getpid()) return else: - logger.info('Started child %s' % pid) + logger.notice('Started child %s' % pid) children.append(pid) try: pid, status = os.wait() @@ -182,8 +182,8 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): if err.errno not in (errno.EINTR, errno.ECHILD): raise except KeyboardInterrupt: - logger.info('User quit') + logger.notice('User quit') break greenio.shutdown_safe(sock) sock.close() - logger.info('Exited') + logger.notice('Exited') diff --git a/swift/proxy/server.py b/swift/proxy/server.py index dc501faba5..c4d8178c61 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -1612,6 +1612,16 @@ class BaseApplication(object): self.logger = logger if conf is None: conf = {} + if 'access_log_name' in conf or 'access_log_facility' in conf: + access_log_conf = { + 'log_name': conf.get('access_log_name', conf.get('log_name', + 'proxy-server')), + 'log_facility': conf.get('access_log_facility', + conf.get('log_facility', 'LOG_LOCAL0')), + } + self.access_logger = get_logger(access_log_conf) + else: + self.access_logger = self.logger swift_dir = conf.get('swift_dir', '/etc/swift') self.node_timeout = int(conf.get('node_timeout', 10)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) @@ -1790,7 +1800,7 @@ class Application(BaseApplication): if getattr(req, 'client_disconnect', False) or \ getattr(response, 'client_disconnect', False): status_int = 499 - self.logger.access(' '.join(quote(str(x)) for x in ( + self.access_logger.info(' '.join(quote(str(x)) for x in ( client or '-', req.remote_addr or '-', time.strftime('%d/%b/%Y/%H/%M/%S', time.gmtime()), diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 1fd18b6ee6..d709c65d3e 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -307,8 +307,8 @@ Error: unable to locate %s logger.debug('test5') self.assertEquals(sio.getvalue(), 'test1\ntest3\ntest4\n') - # make sure access lvl logs by default - logger.access('test6') + # make sure notice lvl logs by default + logger.notice('test7') self.assertEquals(sio.getvalue(), 'test1\ntest3\ntest4\ntest6\n') diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 4b0404454f..9e49b09e74 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1802,11 +1802,12 @@ class TestObjectController(unittest.TestCase): class Logger(object): - def access(self, msg): + def info(self, msg): self.msg = msg orig_logger = prosrv.logger - prosrv.logger = Logger() + orig_access_logger = prosrv.access_logger + prosrv.logger = prosrv.access_logger = Logger() sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write( @@ -1822,11 +1823,13 @@ class TestObjectController(unittest.TestCase): prosrv.logger.msg) exp = 'host1' self.assertEquals(prosrv.logger.msg[:len(exp)], exp) + prosrv.access_logger = orig_access_logger prosrv.logger = orig_logger # Turn on header logging. orig_logger = prosrv.logger - prosrv.logger = Logger() + orig_access_logger = prosrv.access_logger + prosrv.logger = prosrv.access_logger = Logger() prosrv.log_headers = True sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() @@ -1840,6 +1843,7 @@ class TestObjectController(unittest.TestCase): self.assert_('Goofy-Header%3A%20True' in prosrv.logger.msg, prosrv.logger.msg) prosrv.log_headers = False + prosrv.access_logger = orig_access_logger prosrv.logger = orig_logger def test_chunked_put_utf8_all_the_way_down(self): From bd1e2a0daf0fcb1230fda56b746ecc26dbd3100d Mon Sep 17 00:00:00 2001 From: David Goetz Date: Wed, 9 Feb 2011 21:36:14 +0000 Subject: [PATCH 35/64] do not return devs with weight zero in get_more_nodes --- swift/common/ring/ring.py | 10 +++- test/unit/common/ring/test_ring.py | 83 +++++++++++++++++++++--------- 2 files changed, 67 insertions(+), 26 deletions(-) diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 45ab407563..381764a64c 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -139,4 +139,12 @@ class Ring(object): zones.remove(self.devs[part2dev_id[part]]['zone']) while zones: zone = zones.pop(part % len(zones)) - yield self.zone2devs[zone][part % len(self.zone2devs[zone])] + weighted_node = None + for i in xrange(len(self.zone2devs[zone])): + node = self.zone2devs[zone][(part + i) % + len(self.zone2devs[zone])] + if node.get('weight'): + weighted_node = node + break + if weighted_node: + yield weighted_node diff --git a/test/unit/common/ring/test_ring.py b/test/unit/common/ring/test_ring.py index ad72a4c990..1b20480e00 100644 --- a/test/unit/common/ring/test_ring.py +++ b/test/unit/common/ring/test_ring.py @@ -50,7 +50,8 @@ class TestRing(unittest.TestCase): os.mkdir(self.testdir) self.testgz = os.path.join(self.testdir, 'ring.gz') self.intended_replica2part2dev_id = [[0, 2, 0, 2], [2, 0, 2, 0]] - self.intended_devs = [{'id': 0, 'zone': 0}, None, {'id': 2, 'zone': 2}] + self.intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0}, None, + {'id': 2, 'zone': 2, 'weight': 1.0}] self.intended_part_shift = 30 self.intended_reload_time = 15 pickle.dump(ring.RingData(self.intended_replica2part2dev_id, @@ -72,7 +73,7 @@ class TestRing(unittest.TestCase): def test_has_changed(self): self.assertEquals(self.ring.has_changed(), False) - os.utime(self.testgz, (time()+60, time()+60)) + os.utime(self.testgz, (time() + 60, time() + 60)) self.assertEquals(self.ring.has_changed(), True) def test_reload(self): @@ -80,7 +81,7 @@ class TestRing(unittest.TestCase): self.ring = ring.Ring(self.testgz, reload_time=0.001) orig_mtime = self.ring._mtime self.assertEquals(len(self.ring.devs), 3) - self.intended_devs.append({'id': 3, 'zone': 3}) + self.intended_devs.append({'id': 3, 'zone': 3, 'weight': 1.0}) pickle.dump(ring.RingData(self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift), GzipFile(self.testgz, 'wb')) @@ -93,7 +94,7 @@ class TestRing(unittest.TestCase): self.ring = ring.Ring(self.testgz, reload_time=0.001) orig_mtime = self.ring._mtime self.assertEquals(len(self.ring.devs), 4) - self.intended_devs.append({'id': 4, 'zone': 4}) + self.intended_devs.append({'id': 4, 'zone': 4, 'weight': 1.0}) pickle.dump(ring.RingData(self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift), GzipFile(self.testgz, 'wb')) @@ -108,7 +109,7 @@ class TestRing(unittest.TestCase): orig_mtime = self.ring._mtime part, nodes = self.ring.get_nodes('a') self.assertEquals(len(self.ring.devs), 5) - self.intended_devs.append({'id': 5, 'zone': 5}) + self.intended_devs.append({'id': 5, 'zone': 5, 'weight': 1.0}) pickle.dump(ring.RingData(self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift), GzipFile(self.testgz, 'wb')) @@ -127,57 +128,71 @@ class TestRing(unittest.TestCase): self.assertRaises(TypeError, self.ring.get_nodes) part, nodes = self.ring.get_nodes('a') self.assertEquals(part, 0) - self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + self.assertEquals(nodes, [{'id': 0, 'zone': 0, 'weight': 1.0}, + {'id': 2, 'zone': 2, 'weight': 1.0}]) part, nodes = self.ring.get_nodes('a1') self.assertEquals(part, 0) - self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + self.assertEquals(nodes, [{'id': 0, 'zone': 0, 'weight': 1.0}, + {'id': 2, 'zone': 2, 'weight': 1.0}]) part, nodes = self.ring.get_nodes('a4') self.assertEquals(part, 1) - self.assertEquals(nodes, [{'id': 2, 'zone': 2}, {'id': 0, 'zone': 0}]) + self.assertEquals(nodes, [{'id': 2, 'zone': 2, 'weight': 1.0}, + {'id': 0, 'zone': 0, 'weight': 1.0}]) part, nodes = self.ring.get_nodes('aa') self.assertEquals(part, 1) - self.assertEquals(nodes, [{'id': 2, 'zone': 2}, {'id': 0, 'zone': 0}]) + self.assertEquals(nodes, [{'id': 2, 'zone': 2, 'weight': 1.0}, + {'id': 0, 'zone': 0, 'weight': 1.0}]) part, nodes = self.ring.get_nodes('a', 'c1') self.assertEquals(part, 0) - self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + self.assertEquals(nodes, [{'id': 0, 'zone': 0, 'weight': 1.0}, + {'id': 2, 'zone': 2, 'weight': 1.0}]) part, nodes = self.ring.get_nodes('a', 'c0') self.assertEquals(part, 3) - self.assertEquals(nodes, [{'id': 2, 'zone': 2}, {'id': 0, 'zone': 0}]) + self.assertEquals(nodes, [{'id': 2, 'zone': 2, 'weight': 1.0}, + {'id': 0, 'zone': 0, 'weight': 1.0}]) part, nodes = self.ring.get_nodes('a', 'c3') self.assertEquals(part, 2) - self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + self.assertEquals(nodes, [{'id': 0, 'zone': 0, 'weight': 1.0}, + {'id': 2, 'zone': 2, 'weight': 1.0}]) part, nodes = self.ring.get_nodes('a', 'c2') self.assertEquals(part, 2) - self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + self.assertEquals(nodes, [{'id': 0, 'zone': 0, 'weight': 1.0}, + {'id': 2, 'zone': 2, 'weight': 1.0}]) part, nodes = self.ring.get_nodes('a', 'c', 'o1') self.assertEquals(part, 1) - self.assertEquals(nodes, [{'id': 2, 'zone': 2}, {'id': 0, 'zone': 0}]) + self.assertEquals(nodes, [{'id': 2, 'zone': 2, 'weight': 1.0}, + {'id': 0, 'zone': 0, 'weight': 1.0}]) part, nodes = self.ring.get_nodes('a', 'c', 'o5') self.assertEquals(part, 0) - self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + self.assertEquals(nodes, [{'id': 0, 'zone': 0, 'weight': 1.0}, + {'id': 2, 'zone': 2, 'weight': 1.0}]) part, nodes = self.ring.get_nodes('a', 'c', 'o0') self.assertEquals(part, 0) - self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + self.assertEquals(nodes, [{'id': 0, 'zone': 0, 'weight': 1.0}, + {'id': 2, 'zone': 2, 'weight': 1.0}]) part, nodes = self.ring.get_nodes('a', 'c', 'o2') self.assertEquals(part, 2) - self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + self.assertEquals(nodes, [{'id': 0, 'zone': 0, 'weight': 1.0}, + {'id': 2, 'zone': 2, 'weight': 1.0}]) def test_get_more_nodes(self): # Yes, these tests are deliberately very fragile. We want to make sure # that if someone changes the results the ring produces, they know it. part, nodes = self.ring.get_nodes('a', 'c', 'o2') self.assertEquals(part, 2) - self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + self.assertEquals(nodes, [{'id': 0, 'zone': 0, 'weight': 1.0}, + {'id': 2, 'zone': 2, 'weight': 1.0}]) nodes = list(self.ring.get_more_nodes(part)) self.assertEquals(nodes, []) - self.ring.devs.append({'id': 3, 'zone': 0}) + self.ring.devs.append({'id': 3, 'zone': 0, 'weight': 1.0}) self.ring.zone2devs[0].append(self.ring.devs[3]) part, nodes = self.ring.get_nodes('a', 'c', 'o2') self.assertEquals(part, 2) - self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + self.assertEquals(nodes, [{'id': 0, 'zone': 0, 'weight': 1.0}, + {'id': 2, 'zone': 2, 'weight': 1.0}]) nodes = list(self.ring.get_more_nodes(part)) self.assertEquals(nodes, []) @@ -186,18 +201,36 @@ class TestRing(unittest.TestCase): self.ring.zone2devs[3] = [self.ring.devs[3]] part, nodes = self.ring.get_nodes('a', 'c', 'o2') self.assertEquals(part, 2) - self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + self.assertEquals(nodes, [{'id': 0, 'zone': 0, 'weight': 1.0}, + {'id': 2, 'zone': 2, 'weight': 1.0}]) nodes = list(self.ring.get_more_nodes(part)) - self.assertEquals(nodes, [{'id': 3, 'zone': 3}]) + self.assertEquals(nodes, [{'id': 3, 'zone': 3, 'weight': 1.0}]) self.ring.devs.append(None) - self.ring.devs.append({'id': 5, 'zone': 5}) + self.ring.devs.append({'id': 5, 'zone': 5, 'weight': 1.0}) self.ring.zone2devs[5] = [self.ring.devs[5]] part, nodes = self.ring.get_nodes('a', 'c', 'o2') self.assertEquals(part, 2) - self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + self.assertEquals(nodes, [{'id': 0, 'zone': 0, 'weight': 1.0}, + {'id': 2, 'zone': 2, 'weight': 1.0}]) nodes = list(self.ring.get_more_nodes(part)) - self.assertEquals(nodes, [{'id': 3, 'zone': 3}, {'id': 5, 'zone': 5}]) + self.assertEquals(nodes, [{'id': 3, 'zone': 3, 'weight': 1.0}, + {'id': 5, 'zone': 5, 'weight': 1.0}]) + + self.ring.devs.append({'id': 6, 'zone': 5, 'weight': 1.0}) + self.ring.zone2devs[5].append(self.ring.devs[6]) + nodes = list(self.ring.get_more_nodes(part)) + self.assertEquals(nodes, [{'id': 3, 'zone': 3, 'weight': 1.0}, + {'id': 5, 'zone': 5, 'weight': 1.0}]) + self.ring.devs[5]['weight'] = 0 + nodes = list(self.ring.get_more_nodes(part)) + self.assertEquals(nodes, [{'id': 3, 'zone': 3, 'weight': 1.0}, + {'id': 6, 'zone': 5, 'weight': 1.0}]) + self.ring.devs[3]['weight'] = 0 + self.ring.devs.append({'id': 7, 'zone': 6, 'weight': 0.0}) + self.ring.zone2devs[6] = [self.ring.devs[7]] + nodes = list(self.ring.get_more_nodes(part)) + self.assertEquals(nodes, [{'id': 6, 'zone': 5, 'weight': 1.0}]) if __name__ == '__main__': From bb57e753b02cdd115aa9a440aed4597bc2ab20de Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 10 Feb 2011 00:01:07 -0800 Subject: [PATCH 36/64] Fix drive-audit's default log_name --- bin/swift-drive-audit | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/swift-drive-audit b/bin/swift-drive-audit index 5203f54b6b..77912e720e 100755 --- a/bin/swift-drive-audit +++ b/bin/swift-drive-audit @@ -99,6 +99,7 @@ if __name__ == '__main__': device_dir = conf.get('device_dir', '/srv/node') minutes = int(conf.get('minutes', 60)) error_limit = int(conf.get('error_limit', 1)) + conf['log_name'] = conf.get('log_name', 'drive-audit') logger = get_logger(conf, log_route='drive-audit') devices = get_devices(device_dir, logger) logger.debug("Devices found: %s" % str(devices)) From f73d7ad52fafe95fbd06771c0bde8131e004b7d1 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Thu, 10 Feb 2011 10:09:31 -0600 Subject: [PATCH 37/64] Adding python-netifaces to dependencies for packaging docs --- doc/source/debian_package_guide.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/debian_package_guide.rst b/doc/source/debian_package_guide.rst index 4f82f97858..e8086adc16 100644 --- a/doc/source/debian_package_guide.rst +++ b/doc/source/debian_package_guide.rst @@ -107,6 +107,7 @@ Instructions for Deploying Debian Packages for Swift apt-get install rsync python-openssl python-setuptools python-webob python-simplejson python-xattr python-greenlet python-eventlet + python-netifaces #. Install base packages:: From 8193e517af7be246da9e2d3a2aca7b85d3242191 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 10 Feb 2011 11:57:51 -0600 Subject: [PATCH 38/64] slightly more consistant stats process log names Also a quick fix to the auditor tests xattr mock --- bin/swift-account-stats-logger | 2 +- bin/swift-log-uploader | 2 +- swift/common/middleware/catch_errors.py | 6 +----- swift/stats/log_uploader.py | 6 +++--- test/unit/obj/test_auditor.py | 3 ++- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/bin/swift-account-stats-logger b/bin/swift-account-stats-logger index 7b95b20249..6256b690b5 100755 --- a/bin/swift-account-stats-logger +++ b/bin/swift-account-stats-logger @@ -23,4 +23,4 @@ if __name__ == '__main__': # currently AccountStat only supports run_once options['once'] = True run_daemon(AccountStat, conf_file, section_name='log-processor-stats', - **options) + log_name="account-stats", **options) diff --git a/bin/swift-log-uploader b/bin/swift-log-uploader index 93cb8f6f97..7c36e2c2cc 100755 --- a/bin/swift-log-uploader +++ b/bin/swift-log-uploader @@ -34,7 +34,7 @@ if __name__ == '__main__': uploader_conf.update(plugin_conf) # pre-configure logger - logger = utils.get_logger(uploader_conf, plugin, log_route='log-uploader', + logger = utils.get_logger(uploader_conf, log_route='log-uploader', log_to_console=options.get('verbose', False)) # currently LogUploader only supports run_once options['once'] = True diff --git a/swift/common/middleware/catch_errors.py b/swift/common/middleware/catch_errors.py index 16ade84689..716bda4da1 100644 --- a/swift/common/middleware/catch_errors.py +++ b/swift/common/middleware/catch_errors.py @@ -26,11 +26,7 @@ class CatchErrorMiddleware(object): def __init__(self, app, conf): self.app = app - # if the application already has a logger we should use that one - self.logger = getattr(app, 'logger', None) - if not self.logger: - # and only call get_logger if we have to - self.logger = get_logger(conf, log_route='catch-errors') + self.logger = get_logger(conf, log_route='catch-errors') def __call__(self, env, start_response): try: diff --git a/swift/stats/log_uploader.py b/swift/stats/log_uploader.py index a828188eb7..d87d799324 100644 --- a/swift/stats/log_uploader.py +++ b/swift/stats/log_uploader.py @@ -64,9 +64,9 @@ class LogUploader(Daemon): self.container_name = container_name self.filename_format = source_filename_format self.internal_proxy = InternalProxy(proxy_server_conf) - log_name = 'swift-log-uploader-%s' % plugin_name - self.logger = \ - utils.get_logger(uploader_conf, plugin_name, log_route=plugin_name) + log_name = '%s-log-uploader' % plugin_name + self.logger = utils.get_logger(uploader_conf, log_name, + log_route=plugin_name) def run_once(self): self.logger.info(_("Uploading logs")) diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py index 66540a3693..14d58480dd 100644 --- a/test/unit/obj/test_auditor.py +++ b/test/unit/obj/test_auditor.py @@ -14,7 +14,7 @@ # limitations under the License. # TODO: Tests -from test import unit as _setup_mocks +from test import unit import unittest import tempfile import os @@ -57,6 +57,7 @@ class TestAuditor(unittest.TestCase): def tearDown(self): rmtree(os.path.dirname(self.testdir), ignore_errors=1) + unit.xattr_data = {} def test_object_audit_extra_data(self): self.auditor = auditor.ObjectAuditor(self.conf) From 5d0bc6b9c76756a07648f04b4a309677fbec3635 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 10 Feb 2011 14:59:52 -0600 Subject: [PATCH 39/64] logging refactor to support proxy access logs New log level "notice" set to python log level 25 maps to syslog priority LOG_NOTICE. Used for some messages in the proxy server, but will be available to all apps using the LogAdapter returned from get_logger. Cleaned up some code in get_logger so that console logging works with log_routes and removed some unneeded bits. NamedFormatter functionality was split between LogAdapter (which now inherits from logging.LoggerAdapter) and TxnFormatter (which now is only responsible for adding the log records txn_id). The proxy server app now configures a separate logger for access line logging. By default it will use the same settings as the regular proxy logger. --- bin/swift-bench | 8 +- doc/source/overview_stats.rst | 15 +-- swift/common/utils.py | 149 +++++++++------------ swift/proxy/server.py | 24 ++-- test/unit/common/test_utils.py | 234 +++++++++++++++++++++++++++------ test/unit/proxy/test_server.py | 141 ++++++++++++++++++-- 6 files changed, 412 insertions(+), 159 deletions(-) diff --git a/bin/swift-bench b/bin/swift-bench index 3c167ee06f..0554782a06 100755 --- a/bin/swift-bench +++ b/bin/swift-bench @@ -22,7 +22,7 @@ import uuid from optparse import OptionParser from swift.common.bench import BenchController -from swift.common.utils import readconf, LogAdapter, NamedFormatter +from swift.common.utils import readconf, LogAdapter # The defaults should be sufficient to run swift-bench on a SAIO CONF_DEFAULTS = { @@ -125,9 +125,9 @@ if __name__ == '__main__': options.log_level.lower(), logging.INFO)) loghandler = logging.StreamHandler() logger.addHandler(loghandler) - logger = LogAdapter(logger) - logformat = NamedFormatter('swift-bench', logger, - fmt='%(server)s %(asctime)s %(levelname)s %(message)s') + logger = LogAdapter(logger, 'swift-bench') + logformat = logging.Formatter('%(server)s %(asctime)s %(levelname)s ' + '%(message)s') loghandler.setFormatter(logformat) controller = BenchController(logger, options) diff --git a/doc/source/overview_stats.rst b/doc/source/overview_stats.rst index 40a5dd01af..6364de4611 100644 --- a/doc/source/overview_stats.rst +++ b/doc/source/overview_stats.rst @@ -15,10 +15,9 @@ Access logs *********** Access logs are the proxy server logs. Rackspace uses syslog-ng to redirect -proxy log messages with the syslog priority LOG_NOTICE to an hourly log -file. For example, a proxy request that is made on August 4, 2010 at 12:37 gets -logged in a file named 2010080412. This allows easy log rotation and easy -per-hour log processing. +the proxy log output to an hourly log file. For example, a proxy request that +is made on August 4, 2010 at 12:37 gets logged in a file named 2010080412. +This allows easy log rotation and easy per-hour log processing. ****************** Account stats logs @@ -100,11 +99,11 @@ Running the stats system on SAIO destination df_local1 { file("/var/log/swift/proxy.log" owner() group()); }; destination df_local1_err { file("/var/log/swift/proxy.error" owner() group()); }; destination df_local1_hourly { file("/var/log/swift/hourly/$YEAR$MONTH$DAY$HOUR" owner() group()); }; - filter f_local1 { facility(local1) and level(notice); }; + filter f_local1 { facility(local1) and level(info); }; - filter f_local1_err { facility(local1) and not level(notice); }; + filter f_local1_err { facility(local1) and not level(info); }; - # local1.notice -/var/log/swift/proxy.log + # local1.info -/var/log/swift/proxy.log # write to local file and to remove log server log { source(s_all); @@ -182,4 +181,4 @@ earlier. This file will have one entry per account per hour for each account with activity in that hour. One .csv file should be produced per hour. Note that the stats will be delayed by at least two hours by default. This can be changed with the new_log_cutoff variable in the config file. See -`log-processing.conf-sample` for more details. +`log-processing.conf-sample` for more details. \ No newline at end of file diff --git a/swift/common/utils.py b/swift/common/utils.py index 595ad3ac03..3ba291e266 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -289,7 +289,8 @@ class LoggerFileObject(object): return self -class LogAdapter(object): +# double inhereitence to support property with setter +class LogAdapter(logging.LoggerAdapter, object): """ A Logger like object which performs some reformatting on calls to :meth:`exception`. Can be used to store a threadlocal transaction id. @@ -297,11 +298,10 @@ class LogAdapter(object): _txn_id = threading.local() - def __init__(self, logger): - self.logger = logger - for proxied_method in ('debug', 'log', 'warn', 'warning', 'error', - 'critical', 'info'): - setattr(self, proxied_method, getattr(logger, proxied_method)) + def __init__(self, logger, server): + logging.LoggerAdapter.__init__(self, logger, {}) + self.server = server + setattr(self, 'warn', self.warning) @property def txn_id(self): @@ -315,24 +315,34 @@ class LogAdapter(object): def getEffectiveLevel(self): return self.logger.getEffectiveLevel() - def notice(self, msg, *args): + def process(self, msg, kwargs): + """ + Add extra info to message + """ + kwargs['extra'] = {'server': self.server, 'txn_id': self.txn_id} + return msg, kwargs + + def notice(self, msg, *args, **kwargs): """ Convenience function for syslog priority LOG_NOTICE. The python logging lvl is set to 25, just above info. SysLogHandler is monkey patched to map this log lvl to the LOG_NOTICE syslog priority. """ - self.logger.log(NOTICE, msg, *args) + self.log(NOTICE, msg, *args, **kwargs) - def exception(self, msg, *args): + def _exception(self, msg, *args, **kwargs): + logging.LoggerAdapter.exception(self, msg, *args, **kwargs) + + def exception(self, msg, *args, **kwargs): _junk, exc, _junk = sys.exc_info() - call = self.logger.error + call = self.error emsg = '' if isinstance(exc, OSError): if exc.errno in (errno.EIO, errno.ENOSPC): emsg = str(exc) else: - call = self.logger.exception + call = self._exception elif isinstance(exc, socket.error): if exc.errno == errno.ECONNREFUSED: emsg = _('Connection refused') @@ -341,7 +351,7 @@ class LogAdapter(object): elif exc.errno == errno.ETIMEDOUT: emsg = _('Connection timeout') else: - call = self.logger.exception + call = self._exception elif isinstance(exc, eventlet.Timeout): emsg = exc.__class__.__name__ if hasattr(exc, 'seconds'): @@ -350,53 +360,25 @@ class LogAdapter(object): if exc.msg: emsg += ' %s' % exc.msg else: - call = self.logger.exception - call('%s: %s' % (msg, emsg), *args) + call = self._exception + call('%s: %s' % (msg, emsg), *args, **kwargs) -class NamedFormatter(logging.Formatter): +class TxnFormatter(logging.Formatter): """ - NamedFormatter is used to add additional information to log messages. - Normally it will simply add the server name as an attribute on the - LogRecord and the default format string will include it at the - begining of the log message. Additionally, if the transaction id is - available and not already included in the message, NamedFormatter will - add it. - - NamedFormatter may be initialized with a format string which makes use - of the standard LogRecord attributes. In addition the format string - may include the following mapping key: - - +----------------+---------------------------------------------+ - | Format | Description | - +================+=============================================+ - | %(server)s | Name of the swift server doing logging | - +----------------+---------------------------------------------+ - - :param server: the swift server name, a string. - :param logger: a Logger or :class:`LogAdapter` instance, additional - context may be pulled from attributes on this logger if - available. - :param fmt: the format string used to construct the message, if none is - supplied it defaults to ``"%(server)s %(message)s"`` + Custom logging.Formatter will append txn_id to a log message if the record + has one and the message does not. """ - - def __init__(self, server, logger, - fmt="%(server)s %(message)s"): - logging.Formatter.__init__(self, fmt) - self.server = server - self.logger = logger - def format(self, record): - record.server = self.server msg = logging.Formatter.format(self, record) - if self.logger.txn_id and (record.levelno != logging.INFO or - self.logger.txn_id not in msg): - msg = "%s (txn: %s)" % (msg, self.logger.txn_id) + if (record.txn_id and record.levelno != logging.INFO and + record.txn_id not in msg): + msg = "%s (txn: %s)" % (msg, record.txn_id) return msg -def get_logger(conf, name=None, log_to_console=False, log_route=None): +def get_logger(conf, name=None, log_to_console=False, log_route=None, + fmt="%(server)s %(message)s"): """ Get the current system logger using config settings. @@ -412,48 +394,46 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None): """ if not conf: conf = {} - if not hasattr(get_logger, 'root_logger_configured'): - get_logger.root_logger_configured = True - get_logger(conf, name, log_to_console, log_route='root') if name is None: name = conf.get('log_name', 'swift') if not log_route: log_route = name - if log_route == 'root': - logger = logging.getLogger() - else: - logger = logging.getLogger(log_route) - logger.propagate = False - if not hasattr(get_logger, 'handler4facility'): - get_logger.handler4facility = {} - facility = getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'), - SysLogHandler.LOG_LOCAL0) - if facility in get_logger.handler4facility: - logger.removeHandler(get_logger.handler4facility[facility]) - get_logger.handler4facility[facility].close() - del get_logger.handler4facility[facility] - if log_to_console: - # check if a previous call to get_logger already added a console logger - if hasattr(get_logger, 'console') and get_logger.console: - logger.removeHandler(get_logger.console) - get_logger.console = logging.StreamHandler(sys.__stderr__) - logger.addHandler(get_logger.console) - get_logger.handler4facility[facility] = \ - SysLogHandler(address='/dev/log', facility=facility) + logger = logging.getLogger(log_route) + logger.propagate = False + # all swift new handlers will get the same formatter + formatter = TxnFormatter(fmt) + + # a single swift logger will only get one SysLog Handler if not hasattr(get_logger, 'handler4logger'): get_logger.handler4logger = {} if logger in get_logger.handler4logger: logger.removeHandler(get_logger.handler4logger[logger]) - get_logger.handler4logger[logger] = \ - get_logger.handler4facility[facility] - logger.addHandler(get_logger.handler4facility[facility]) + + # facility for this logger will be set by last call wins + facility = getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'), + SysLogHandler.LOG_LOCAL0) + handler = SysLogHandler(address='/dev/log', facility=facility) + handler.setFormatter(formatter) + logger.addHandler(handler) + get_logger.handler4logger[logger] = handler + + # setup console logging + if log_to_console or hasattr(get_logger, 'console_handler4logger'): + # remove pre-existing console handler for this logger + if not hasattr(get_logger, 'console_handler4logger'): + get_logger.console_handler4logger = {} + if logger in get_logger.console_handler4logger: + logger.removeHandler(get_logger.console_handler4logger[logger]) + + console_handler = logging.StreamHandler(sys.__stderr__) + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + get_logger.console_handler4logger[logger] = console_handler + + # set the level for the logger logger.setLevel( getattr(logging, conf.get('log_level', 'INFO').upper(), logging.INFO)) - adapted_logger = LogAdapter(logger) - formatter = NamedFormatter(name, adapted_logger) - get_logger.handler4facility[facility].setFormatter(formatter) - if hasattr(get_logger, 'console'): - get_logger.console.setFormatter(formatter) + adapted_logger = LogAdapter(logger, name) return adapted_logger @@ -486,8 +466,9 @@ def capture_stdio(logger, **kwargs): # collect stdio file desc not in use for logging stdio_fds = [0, 1, 2] - if hasattr(get_logger, 'console'): - stdio_fds.remove(get_logger.console.stream.fileno()) + for _junk, handler in getattr(get_logger, + 'console_handler4logger', {}).items(): + stdio_fds.remove(handler.stream.fileno()) with open(os.devnull, 'r+b') as nullfile: # close stdio (excludes fds open for logging) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index c4d8178c61..b2b41d6434 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -1606,22 +1606,20 @@ class BaseApplication(object): def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, object_ring=None): - if logger is None: - self.logger = get_logger(conf) - else: - self.logger = logger if conf is None: conf = {} - if 'access_log_name' in conf or 'access_log_facility' in conf: - access_log_conf = { - 'log_name': conf.get('access_log_name', conf.get('log_name', - 'proxy-server')), - 'log_facility': conf.get('access_log_facility', - conf.get('log_facility', 'LOG_LOCAL0')), - } - self.access_logger = get_logger(access_log_conf) + if logger is None: + self.logger = get_logger(conf) + access_log_conf = {} + for key in ('log_facility', 'log_name', 'log_level'): + value = conf.get('access_' + key, conf.get(key, None)) + if value: + access_log_conf[key] = value + self.access_logger = get_logger(access_log_conf, + log_route='proxy-access') else: - self.access_logger = self.logger + self.logger = self.access_logger = logger + swift_dir = conf.get('swift_dir', '/etc/swift') self.node_timeout = int(conf.get('node_timeout', 10)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index d709c65d3e..959caa8919 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -19,6 +19,7 @@ from __future__ import with_statement import logging import mimetools import os +import errno import socket import sys import time @@ -31,6 +32,8 @@ from tempfile import NamedTemporaryFile from eventlet import sleep +from swift.common.exceptions import TimeoutError, MessageTimeout, \ + ConnectionTimeout from swift.common import utils @@ -76,6 +79,17 @@ class MockSys(): __stderr__ = sys.__stderr__ +def reset_loggers(): + if hasattr(utils.get_logger, 'handler4logger'): + for logger, handler in utils.get_logger.handler4logger.items(): + logger.removeHandler(handler) + delattr(utils.get_logger, 'handler4logger') + if hasattr(utils.get_logger, 'console_handler4logger'): + for logger, h in utils.get_logger.console_handler4logger.items(): + logger.removeHandler(h) + delattr(utils.get_logger, 'console_handler4logger') + + class TestUtils(unittest.TestCase): """ Tests for swift.common.utils """ @@ -308,10 +322,131 @@ Error: unable to locate %s self.assertEquals(sio.getvalue(), 'test1\ntest3\ntest4\n') # make sure notice lvl logs by default - logger.notice('test7') - self.assertEquals(sio.getvalue(), - 'test1\ntest3\ntest4\ntest6\n') + logger.notice('test6') + def test_clean_logger_exception(self): + # setup stream logging + sio = StringIO() + logger = utils.get_logger(None) + handler = logging.StreamHandler(sio) + logger.logger.addHandler(handler) + + def strip_value(sio): + v = sio.getvalue() + sio.truncate(0) + return v + + def log_exception(exc): + try: + raise exc + except (Exception, TimeoutError): + logger.exception('blah') + try: + # establish base case + self.assertEquals(strip_value(sio), '') + logger.info('test') + self.assertEquals(strip_value(sio), 'test\n') + self.assertEquals(strip_value(sio), '') + logger.info('test') + logger.info('test') + self.assertEquals(strip_value(sio), 'test\ntest\n') + self.assertEquals(strip_value(sio), '') + + # test OSError + for en in (errno.EIO, errno.ENOSPC): + log_exception(OSError(en, 'my %s error message' % en)) + log_msg = strip_value(sio) + self.assert_('Traceback' not in log_msg) + self.assert_('my %s error message' % en in log_msg) + # unfiltered + log_exception(OSError()) + self.assert_('Traceback' in strip_value(sio)) + + # test socket.error + log_exception(socket.error(errno.ECONNREFUSED, + 'my error message')) + log_msg = strip_value(sio) + self.assert_('Traceback' not in log_msg) + self.assert_('errno.ECONNREFUSED message test' not in log_msg) + self.assert_('Connection refused' in log_msg) + log_exception(socket.error(errno.EHOSTUNREACH, + 'my error message')) + log_msg = strip_value(sio) + self.assert_('Traceback' not in log_msg) + self.assert_('my error message' not in log_msg) + self.assert_('Host unreachable' in log_msg) + log_exception(socket.error(errno.ETIMEDOUT, 'my error message')) + log_msg = strip_value(sio) + self.assert_('Traceback' not in log_msg) + self.assert_('my error message' not in log_msg) + self.assert_('Connection timeout' in log_msg) + # unfiltered + log_exception(socket.error(0, 'my error message')) + log_msg = strip_value(sio) + self.assert_('Traceback' in log_msg) + self.assert_('my error message' in log_msg) + + # test eventlet.Timeout + log_exception(ConnectionTimeout(42, 'my error message')) + log_msg = strip_value(sio) + self.assert_('Traceback' not in log_msg) + self.assert_('ConnectionTimeout' in log_msg) + self.assert_('(42s)' in log_msg) + self.assert_('my error message' not in log_msg) + log_exception(MessageTimeout(42, 'my error message')) + log_msg = strip_value(sio) + self.assert_('Traceback' not in log_msg) + self.assert_('MessageTimeout' in log_msg) + self.assert_('(42s)' in log_msg) + self.assert_('my error message' in log_msg) + + # test unhandled + log_exception(Exception('my error message')) + log_msg = strip_value(sio) + self.assert_('Traceback' in log_msg) + self.assert_('my error message' in log_msg) + + finally: + logger.logger.removeHandler(handler) + reset_loggers() + + def test_txn_formatter(self): + # setup stream logging + sio = StringIO() + logger = utils.get_logger(None) + handler = logging.StreamHandler(sio) + handler.setFormatter(utils.TxnFormatter()) + logger.logger.addHandler(handler) + + def strip_value(sio): + v = sio.getvalue() + sio.truncate(0) + return v + + try: + self.assertFalse(logger.txn_id) + logger.error('my error message') + log_msg = strip_value(sio) + self.assert_('my error message' in log_msg) + self.assert_('txn' not in log_msg) + logger.txn_id = '12345' + logger.error('test') + log_msg = strip_value(sio) + self.assert_('txn' in log_msg) + self.assert_('12345' in log_msg) + # test no txn on info message + self.assertEquals(logger.txn_id, '12345') + logger.info('test') + log_msg = strip_value(sio) + self.assert_('txn' not in log_msg) + self.assert_('12345' not in log_msg) + # test txn already in message + self.assertEquals(logger.txn_id, '12345') + logger.warn('test 12345 test') + self.assertEquals(strip_value(sio), 'test 12345 test\n') + finally: + logger.logger.removeHandler(handler) + reset_loggers() def test_storage_directory(self): self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'), @@ -397,56 +532,71 @@ log_name = yarr''' logger = utils.get_logger(None, 'dummy') # mock utils system modules - utils.sys = MockSys() - utils.os = MockOs() + _orig_sys = utils.sys + _orig_os = utils.os + try: + utils.sys = MockSys() + utils.os = MockOs() - # basic test - utils.capture_stdio(logger) - self.assert_(utils.sys.excepthook is not None) - self.assertEquals(utils.os.closed_fds, [0, 1, 2]) - self.assert_(utils.sys.stdout is not None) - self.assert_(utils.sys.stderr is not None) + # basic test + utils.capture_stdio(logger) + self.assert_(utils.sys.excepthook is not None) + self.assertEquals(utils.os.closed_fds, [0, 1, 2]) + self.assert_(utils.sys.stdout is not None) + self.assert_(utils.sys.stderr is not None) - # reset; test same args, but exc when trying to close stdio - utils.os = MockOs(raise_funcs=('dup2',)) - utils.sys = MockSys() + # reset; test same args, but exc when trying to close stdio + utils.os = MockOs(raise_funcs=('dup2',)) + utils.sys = MockSys() - # test unable to close stdio - utils.capture_stdio(logger) - self.assert_(utils.sys.excepthook is not None) - self.assertEquals(utils.os.closed_fds, []) - self.assert_(utils.sys.stdout is not None) - self.assert_(utils.sys.stderr is not None) + # test unable to close stdio + utils.capture_stdio(logger) + self.assert_(utils.sys.excepthook is not None) + self.assertEquals(utils.os.closed_fds, []) + self.assert_(utils.sys.stdout is not None) + self.assert_(utils.sys.stderr is not None) - # reset; test some other args - logger = utils.get_logger(None, log_to_console=True) - utils.os = MockOs() - utils.sys = MockSys() + # reset; test some other args + logger = utils.get_logger(None, log_to_console=True) + utils.os = MockOs() + utils.sys = MockSys() - # test console log - utils.capture_stdio(logger, capture_stdout=False, - capture_stderr=False) - self.assert_(utils.sys.excepthook is not None) - # when logging to console, stderr remains open - self.assertEquals(utils.os.closed_fds, [0, 1]) - logger.logger.removeHandler(utils.get_logger.console) - # stdio not captured - self.assertFalse(hasattr(utils.sys, 'stdout')) - self.assertFalse(hasattr(utils.sys, 'stderr')) + # test console log + utils.capture_stdio(logger, capture_stdout=False, + capture_stderr=False) + self.assert_(utils.sys.excepthook is not None) + # when logging to console, stderr remains open + self.assertEquals(utils.os.closed_fds, [0, 1]) + reset_loggers() + + # stdio not captured + self.assertFalse(hasattr(utils.sys, 'stdout')) + self.assertFalse(hasattr(utils.sys, 'stderr')) + reset_loggers() + finally: + utils.sys = _orig_sys + utils.os = _orig_os def test_get_logger_console(self): - reload(utils) # reset get_logger attrs + reset_loggers() logger = utils.get_logger(None) - self.assertFalse(hasattr(utils.get_logger, 'console')) + console_handlers = [h for h in logger.logger.handlers if + isinstance(h, logging.StreamHandler)] + self.assertFalse(console_handlers) logger = utils.get_logger(None, log_to_console=True) - self.assert_(hasattr(utils.get_logger, 'console')) - self.assert_(isinstance(utils.get_logger.console, - logging.StreamHandler)) + console_handlers = [h for h in logger.logger.handlers if + isinstance(h, logging.StreamHandler)] + self.assert_(console_handlers) # make sure you can't have two console handlers - old_handler = utils.get_logger.console + self.assertEquals(len(console_handlers), 1) + old_handler = console_handlers[0] logger = utils.get_logger(None, log_to_console=True) - self.assertNotEquals(utils.get_logger.console, old_handler) - logger.logger.removeHandler(utils.get_logger.console) + console_handlers = [h for h in logger.logger.handlers if + isinstance(h, logging.StreamHandler)] + self.assertEquals(len(console_handlers), 1) + new_handler = console_handlers[0] + self.assertNotEquals(new_handler, old_handler) + reset_loggers() def test_ratelimit_sleep(self): running_time = 0 diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 9e49b09e74..71b61e1e2c 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -16,6 +16,7 @@ from __future__ import with_statement import cPickle as pickle import logging +from logging.handlers import SysLogHandler import os import sys import unittest @@ -465,8 +466,138 @@ class TestController(unittest.TestCase): test(404, 507, 503) test(503, 503, 503) + class TestProxyServer(unittest.TestCase): + def test_access_log(self): + + class MyApp(proxy_server.Application): + + def handle_request(self, req): + resp = Response(request=req) + req.response = resp + req.start_time = time() + return resp + + def start_response(*args): + pass + + class MockLogger(): + + def __init__(self): + self.buffer = StringIO() + + def info(self, msg, args=None): + if args: + msg = msg % args + self.buffer.write(msg) + + def strip_value(self): + rv = self.buffer.getvalue() + self.buffer.truncate(0) + return rv + + class SnarfStream(object): + # i can't seem to subclass cStringIO + + def __init__(self, *args, **kwargs): + self.sio = StringIO() + + def strip_value(self): + rv = self.getvalue().strip() + self.truncate(0) + return rv + + def __getattr__(self, name): + try: + return object.__getattr__(self, name) + except AttributeError: + try: + return getattr(self.sio, name) + except AttributeError: + return self.__getattribute__(name) + + snarf = SnarfStream() + _orig_get_logger = proxy_server.get_logger + + def mock_get_logger(*args, **kwargs): + if kwargs.get('log_route') != 'proxy-access': + return _orig_get_logger(*args, **kwargs) + kwargs['log_route'] = 'snarf' + logger = _orig_get_logger(*args, **kwargs) + if [h for h in logger.logger.handlers if + isinstance(h, logging.StreamHandler) and h.stream is snarf]: + # snarf handler already setup! + return logger + formatter = logger.logger.handlers[0].formatter + formatter._fmt += ' %(levelname)s' + snarf_handler = logging.StreamHandler(snarf) + snarf_handler.setFormatter(formatter) + logger.logger.addHandler(snarf_handler) + return logger + + def test_conf(conf): + app = MyApp(conf, memcache=FakeMemcache(), account_ring=FakeRing(), + container_ring=FakeRing(), object_ring=FakeRing()) + req = Request.blank('') + app(req.environ, start_response) + + try: + proxy_server.get_logger = mock_get_logger + test_conf({}) + line = snarf.strip_value() + print line + self.assert_(line.startswith('swift')) + self.assert_(line.endswith('INFO')) + test_conf({'log_name': 'snarf-test'}) + line = snarf.strip_value() + print line + self.assert_(line.startswith('snarf-test')) + self.assert_(line.endswith('INFO')) + test_conf({'log_name': 'snarf-test', 'log_level': 'ERROR'}) + line = snarf.strip_value() + print line + self.assertFalse(line) + test_conf({'log_name': 'snarf-test', 'log_level': 'ERROR', + 'access_log_name': 'access-test', + 'access_log_level': 'INFO'}) + line = snarf.strip_value() + print line + self.assert_(line.startswith('access-test')) + self.assert_(line.endswith('INFO')) + + # test facility + def get_facility(logger): + h = [h for h in logger.logger.handlers if + isinstance(h, SysLogHandler)][0] + return h.facility + + conf = {'log_facility': 'LOG_LOCAL0'} + app = MyApp(conf, memcache=FakeMemcache(), account_ring=FakeRing(), + container_ring=FakeRing(), object_ring=FakeRing()) + self.assertEquals(get_facility(app.logger), + SysLogHandler.LOG_LOCAL0) + self.assertEquals(get_facility(app.access_logger), + SysLogHandler.LOG_LOCAL0) + conf = {'log_facility': 'LOG_LOCAL0', + 'access_log_facility': 'LOG_LOCAL1'} + app = MyApp(conf, memcache=FakeMemcache(), account_ring=FakeRing(), + container_ring=FakeRing(), object_ring=FakeRing()) + self.assertEquals(get_facility(app.logger), + SysLogHandler.LOG_LOCAL0) + self.assertEquals(get_facility(app.access_logger), + SysLogHandler.LOG_LOCAL1) + conf = {'access_log_facility': 'LOG_LOCAL1'} + app = MyApp(conf, memcache=FakeMemcache(), account_ring=FakeRing(), + container_ring=FakeRing(), object_ring=FakeRing()) + self.assertEquals(get_facility(app.logger), + SysLogHandler.LOG_LOCAL0) + self.assertEquals(get_facility(app.access_logger), + SysLogHandler.LOG_LOCAL1) + + finally: + proxy_server.get_logger = _orig_get_logger + def test_unhandled_exception(self): class MyApp(proxy_server.Application): @@ -1805,8 +1936,7 @@ class TestObjectController(unittest.TestCase): def info(self, msg): self.msg = msg - orig_logger = prosrv.logger - orig_access_logger = prosrv.access_logger + orig_logger, orig_access_logger = prosrv.logger, prosrv.access_logger prosrv.logger = prosrv.access_logger = Logger() sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() @@ -1823,12 +1953,8 @@ class TestObjectController(unittest.TestCase): prosrv.logger.msg) exp = 'host1' self.assertEquals(prosrv.logger.msg[:len(exp)], exp) - prosrv.access_logger = orig_access_logger - prosrv.logger = orig_logger # Turn on header logging. - orig_logger = prosrv.logger - orig_access_logger = prosrv.access_logger prosrv.logger = prosrv.access_logger = Logger() prosrv.log_headers = True sock = connect_tcp(('localhost', prolis.getsockname()[1])) @@ -1843,8 +1969,7 @@ class TestObjectController(unittest.TestCase): self.assert_('Goofy-Header%3A%20True' in prosrv.logger.msg, prosrv.logger.msg) prosrv.log_headers = False - prosrv.access_logger = orig_access_logger - prosrv.logger = orig_logger + prosrv.logger, prosrv.access_logger = orig_logger, orig_access_logger def test_chunked_put_utf8_all_the_way_down(self): # Test UTF-8 Unicode all the way through the system From 5082b6d38958234351756757385f0301ff704087 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 10 Feb 2011 15:05:53 -0600 Subject: [PATCH 40/64] updated proxy-server.conf-sample to include access_log_* defaults --- etc/proxy-server.conf-sample | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index fad511ca30..3af7db0f8a 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -24,6 +24,9 @@ use = egg:swift#proxy # set log_name = proxy-server # set log_facility = LOG_LOCAL0 # set log_level = INFO +# set access_log_name = proxy-server +# set access_log_facility = LOG_LOCAL0 +# set access_log_level = INFO # set log_headers = False # recheck_account_existence = 60 # recheck_container_existence = 60 From c973bf53fb08efd93943511f367b1fb5075fde41 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 10 Feb 2011 15:23:59 -0600 Subject: [PATCH 41/64] cleaned up some comments --- swift/common/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index 3ba291e266..4df8b624bc 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -51,7 +51,6 @@ logging._lock = logging.threading.RLock() # setup notice level logging NOTICE = 25 logging._levelNames[NOTICE] = 'NOTICE' -# syslog priority "notice" is used for proxy access log lines SysLogHandler.priority_map['NOTICE'] = 'notice' # These are lazily pulled from libc elsewhere @@ -289,7 +288,7 @@ class LoggerFileObject(object): return self -# double inhereitence to support property with setter +# double inheritance to support property with setter class LogAdapter(logging.LoggerAdapter, object): """ A Logger like object which performs some reformatting on calls to @@ -391,6 +390,7 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None, :param conf: Configuration dict to read settings from :param name: Name of the logger :param log_to_console: Add handler which writes to console on stderr + :param fmt: Override log format """ if not conf: conf = {} @@ -400,10 +400,10 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None, log_route = name logger = logging.getLogger(log_route) logger.propagate = False - # all swift new handlers will get the same formatter + # all new handlers will get the same formatter formatter = TxnFormatter(fmt) - # a single swift logger will only get one SysLog Handler + # get_logger will only ever add one SysLog Handler to a logger if not hasattr(get_logger, 'handler4logger'): get_logger.handler4logger = {} if logger in get_logger.handler4logger: From 0c0920701a95d0b96c3e50dff31f7b1acc81b905 Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 10 Feb 2011 15:10:53 -0800 Subject: [PATCH 42/64] PEP8 Fixes --- swift/common/db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index 83cd0e8188..9f322e7b7d 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -287,7 +287,7 @@ class DatabaseBroker(object): self.conn = None orig_isolation_level = conn.isolation_level conn.isolation_level = None - conn.execute('PRAGMA journal_mode = DELETE') # remove any journal files + conn.execute('PRAGMA journal_mode = DELETE') # remove journal files conn.execute('BEGIN IMMEDIATE') try: yield True @@ -295,7 +295,7 @@ class DatabaseBroker(object): pass try: conn.execute('ROLLBACK') - conn.execute('PRAGMA journal_mode = WAL') # back to WAL mode + conn.execute('PRAGMA journal_mode = WAL') # back to WAL mode conn.isolation_level = orig_isolation_level self.conn = conn except Exception: From 1095f27590d52f85bfb89c3f503155b914c979ea Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Fri, 11 Feb 2011 13:18:19 -0600 Subject: [PATCH 43/64] Prepare for trunk merge. Refactored some of the swift_init classes into a new module in swift.common, changed some names. Removed the bin test stuff. Fixed some bugs, added some features. --- bin/swift-init | 651 +-------------- swift/common/utils.py | 53 ++ swift/common/wsgi.py | 1 + test/bin/__init__.py | 32 - test/bin/test_swift_init.py | 1414 -------------------------------- test/unit/__init__.py | 23 + test/unit/common/test_utils.py | 82 ++ 7 files changed, 169 insertions(+), 2087 deletions(-) delete mode 100644 test/bin/__init__.py delete mode 100644 test/bin/test_swift_init.py diff --git a/bin/swift-init b/bin/swift-init index ce72242c0d..ff7cc80285 100755 --- a/bin/swift-init +++ b/bin/swift-init @@ -14,646 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import with_statement -import functools -import errno -import glob -import os -import resource -import signal import sys -import time from optparse import OptionParser -import subprocess - -from swift.common import utils - -SWIFT_DIR = '/etc/swift' -RUN_DIR = '/var/run/swift' - -ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor', - 'container-replicator', 'container-server', 'container-updater', - 'object-auditor', 'object-server', 'object-replicator', 'object-updater', - 'proxy-server', 'account-replicator', 'auth-server', 'account-reaper'] -MAIN_SERVERS = ['auth-server', 'proxy-server', 'account-server', - 'container-server', 'object-server'] -REST_SERVERS = [s for s in ALL_SERVERS if s not in MAIN_SERVERS] -GRACEFUL_SHUTDOWN_SERVERS = MAIN_SERVERS -START_ONCE_SERVERS = REST_SERVERS - -KILL_WAIT = 15 # seconds to wait for servers to die - -MAX_DESCRIPTORS = 32768 -MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB - - -def setup_env(): - """Try to increase resource limits of the OS. Move PYTHON_EGG_CACHE to /tmp - """ - try: - resource.setrlimit(resource.RLIMIT_NOFILE, - (MAX_DESCRIPTORS, MAX_DESCRIPTORS)) - resource.setrlimit(resource.RLIMIT_DATA, - (MAX_MEMORY, MAX_MEMORY)) - except ValueError: - print "WARNING: Unable to increase file descriptor limit." \ - " Running as non-root?" - - os.environ['PYTHON_EGG_CACHE'] = '/tmp' - return - - -def search_tree(root, glob_match, ext): - """Look in root, for any files/dirs matching glob, recurively traversing - any found directories looking for files ending with ext - - :param root: start of search path - :param glob_match: glob to match in root, matching dirs are traversed with - os.walk - :param ext: only files that end in ext will be returned - - :returns: list of full paths to matching files, sorted - - """ - found_files = [] - for path in glob.glob(os.path.join(root, glob_match)): - if path.endswith(ext): - found_files.append(path) - else: - for root, dirs, files in os.walk(path): - for file in files: - if file.endswith(ext): - found_files.append(os.path.join(root, file)) - return sorted(found_files) - - -def write_file(path, contents): - """Write contents to file at path - - :param path: any path, subdirs will be created as needed - :param contents: data to write to file, will be converted to string - - """ - dir, name = os.path.split(path) - if not os.path.exists(dir): - try: - os.makedirs(dir) - except OSError, err: - if err.errno == errno.EACCES: - sys.exit('Unable to create %s. Running as non-root?' % dir) - with open(path, 'w') as f: - f.write('%s' % contents) - - -def remove_file(path): - """Quiet wrapper for os.unlink, OSErrors are suppressed - - :param path: first and only argument passed to os.unlink - """ - try: - os.unlink(path) - except OSError: - pass - - -def command(func): - """ - Decorator to declare which methods are accessible as commands, commands - always return 1 or 0, where 0 should indicate success. - - :param func: function to make public - """ - func.publicly_accessible = True - - @functools.wraps(func) - def wrapped(*a, **kw): - rv = func(*a, **kw) - return 1 if rv else 0 - return wrapped - - -class UnknownCommand(Exception): - pass - - -class SwiftInit(): - """Main class for performing commands on groups of servers. - - :param servers: list of server names as strings - - """ - - def __init__(self, servers): - server_names = set() - for server in servers: - if server == 'all': - server_names.update(ALL_SERVERS) - elif server == 'main': - server_names.update(MAIN_SERVERS) - elif server == 'rest': - server_names.update(REST_SERVERS) - else: - server_names.add(server) - - self.servers = set() - for name in server_names: - self.servers.add(SwiftServer(name)) - - def watch_server_pids(self, server_pids, interval=0, **kwargs): - """Monitor a collection of server pids yeilding back those pids that - aren't responding to signals. - - :param server_pids: a dict, lists of pids [int,...] keyed on - SwiftServer objects - """ - status = {} - start = time.time() - end = start + interval - while interval: - for server, pids in server_pids.items(): - for pid in pids: - # let pid stop if it wants to - try: - os.waitpid(pid, os.WNOHANG) - except OSError, e: - if e.errno in (errno.ECHILD, errno.ESRCH): - pass - else: - raise - status[server] = server.get_running_pids(**kwargs) - #print server, pids, status[server] - for pid in pids: - if pid not in status[server]: - yield server, pid - server_pids[server] = status[server] - if not [p for server, pids in status.items() for p in pids]: - # no more running pids - break - if time.time() > end: - break - else: - time.sleep(0.1) - return - - @command - def status(self, **kwargs): - """display status of tracked pids for server - """ - status = 0 - for server in self.servers: - status += server.status(**kwargs) - return status - - @command - def start(self, **kwargs): - """starts a server - """ - setup_env() - status = 0 - - for server in self.servers: - server.launch(**kwargs) - if kwargs.get('wait', False): - for server in self.servers: - status += server.wait(**kwargs) - if not kwargs.get('daemon', True): - for server in self.servers: - try: - status += server.interact(**kwargs) - except KeyboardInterrupt: - print '\nuser quit' - self.stop(**kwargs) - break - return status - - @command - def wait(self, **kwargs): - """spawn server and wait for it to start - """ - kwargs['wait'] = True - return self.start(**kwargs) - - @command - def no_daemon(self, **kwargs): - """start a server interactivly - """ - kwargs['daemon'] = False - return self.start(**kwargs) - - @command - def once(self, **kwargs): - """start server and run one pass on supporting daemons - """ - kwargs['once'] = True - return self.start(**kwargs) - - @command - def stop(self, **kwargs): - """stops a server - """ - server_pids = {} - for server in self.servers: - signaled_pids = server.stop(**kwargs) - if not signaled_pids: - print 'No %s running' % server - else: - server_pids[server] = signaled_pids - - # all signaled_pids, i.e. list(itertools.chain(*server_pids.values())) - signaled_pids = [p for server, pid in server_pids.items() for p in pid] - # keep track of the pids yeiled back as killed for all servers - killed_pids = set() - for server, killed_pid in self.watch_server_pids(server_pids, - interval=KILL_WAIT, **kwargs): - print "%s (%s) appears to have stopped" % (server, killed_pid) - killed_pids.add(killed_pid) - if not killed_pids.symmetric_difference(signaled_pids): - # all proccesses have been stopped - return 0 - - # reached interval n watch_pids w/o killing all servers - for server, pids in server_pids.items(): - if not killed_pids.issuperset(pids): - # some pids of this server were not killed - print 'Waited 15 seconds for %s to die; giving up' % (server) - return 1 - - @command - def shutdown(self, **kwargs): - """allow current requests to finish on supporting servers - """ - kwargs['graceful'] = True - status = 0 - self.stop(**kwargs) - return status - - @command - def restart(self, **kwargs): - """stops then restarts server - """ - status = 0 - self.stop(**kwargs) - self.start(**kwargs) - return status - - @command - def reload(self, **kwargs): - """graceful shutdown then restart on supporting servers - """ - kwargs['graceful'] = True - status = 0 - for server in self.servers: - init = SwiftInit([server.server]) - status += init.stop(**kwargs) - status += init.start(**kwargs) - return status - - @command - def force_reload(self, **kwargs): - """alias for reload - """ - return self.reload(**kwargs) - - def get_command(self, cmd): - """Find and return the decorated method named like cmd - - :param cmd: the command to get, a string, if not found raises - UnknownCommand - - """ - cmd = cmd.lower().replace('-', '_') - try: - f = getattr(self, cmd) - except AttributeError: - raise UnknownCommand(cmd) - if not hasattr(f, 'publicly_accessible'): - raise UnknownCommand(cmd) - return f - - @classmethod - def list_commands(cls): - """Get all publicly accessible commands - - :returns: a list of strings, the method names who are decorated - as commands - """ - get_method = lambda cmd: getattr(cls, cmd) - return sorted([(x.replace('_', '-'), get_method(x).__doc__.strip()) - for x in dir(cls) if - getattr(get_method(x), 'publicly_accessible', False)]) - - def run_command(self, cmd, **kwargs): - """Find the named command and run it - - :param cmd: the command name to run - - """ - f = self.get_command(cmd) - return f(**kwargs) - - -class SwiftServer(): - """Manage operations on a server or group of servers of similar type - - :param server: name of server - """ - - def __init__(self, server): - if '-' not in server: - server = '%s-server' % server - self.server = server.lower() - self.type = '-'.join(server.split('-')[:-1]) - self.cmd = 'swift-%s' % server - self.procs = [] - - def __str__(self): - return self.server - - def __repr__(self): - return "%s(%s)" % (self.__class__.__name__, repr(str(self))) - - def __hash__(self): - return hash(str(self)) - - def __eq__(self, other): - try: - return self.server == other.server - except AttributeError: - return False - - def get_pid_file_name(self, ini_file): - """Translate ini_file to a corresponding pid_file - - :param ini_file: an ini_file for this server, a string - - :returns: the pid_file for this ini_file - - """ - return ini_file.replace( - os.path.normpath(SWIFT_DIR), RUN_DIR, 1).replace( - '%s-server' % self.type, self.server, 1).rsplit( - '.conf', 1)[0] + '.pid' - - def get_ini_file_name(self, pid_file): - """Translate pid_file to a corresponding ini_file - - :param pid_file: a pid_file for this server, a string - - :returns: the ini_file for this pid_file - - """ - return pid_file.replace( - os.path.normpath(RUN_DIR), SWIFT_DIR, 1).replace( - self.server, '%s-server' % self.type, 1).rsplit( - '.pid', 1)[0] + '.conf' - - def ini_files(self, **kwargs): - """Get ini files for this server - - :param: number, if supplied will only lookup the nth server - - :returns: list of ini files - """ - found_ini_files = search_tree(SWIFT_DIR, '%s-server*' % self.type, - '.conf') - number = kwargs.get('number') - if number: - try: - ini_files = [found_ini_files[number - 1]] - except IndexError: - ini_files = [] - else: - ini_files = found_ini_files - if not ini_files: - # maybe there's a config file(s) out there, but I couldn't find it! - if not kwargs.get('quiet'): - print('Unable to locate config %sfor %s' % ( - ('number %s ' % number if number else ''), self.server)) - if kwargs.get('verbose') and not kwargs.get('quiet'): - if found_ini_files: - print('Found configs:') - for i, ini_file in enumerate(found_ini_files): - print(' %d) %s' % (i + 1, ini_file)) - - return ini_files - - def pid_files(self, **kwargs): - """Get pid files for this server - - :param: number, if supplied will only lookup the nth server - - :returns: list of pid files - """ - pid_files = search_tree(RUN_DIR, '%s*' % self.server, '.pid') - number = kwargs.get('number', 0) - if number: - ini_files = self.ini_files(**kwargs) - # limt pid_files the one who translates to the indexed ini_file for - # this given number - pid_files = [pid_file for pid_file in pid_files if - self.get_ini_file_name(pid_file) in ini_files] - return pid_files - - def iter_pid_files(self, **kwargs): - """Generator, yields (pid_file, pids) - """ - for pid_file in self.pid_files(**kwargs): - yield pid_file, int(open(pid_file).read().strip()) - - def signal_pids(self, sig, **kwargs): - """Send a signal to pids for this server - - :param sig: signal to send - - :returns: a dict mapping pids (ints) to pid_files (paths) - - """ - pids = {} - for pid_file, pid in self.iter_pid_files(**kwargs): - try: - if sig != signal.SIG_DFL: - print 'Signal %s pid: %s signal: %s' % ( - self.server, pid, sig) - os.kill(pid, sig) - except OSError, e: - #print '%s sig err: %s' % (pid, e) - if e.errno == 3: - # pid does not exist - if kwargs.get('verbose'): - print "Removing stale pid file %s" % pid_file - remove_file(pid_file) - else: - # process exists - pids[pid] = pid_file - return pids - - def get_running_pids(self, **kwargs): - """Get running pids - - :returns: a dict mapping pids (ints) to pid_files (paths) - - """ - return self.signal_pids(signal.SIG_DFL, **kwargs) # send noop - - def kill_running_pids(self, **kwargs): - """Kill running pids - - :param graceful: if True, attempt SIGHUP on supporting servers - - :returns: a dict mapping pids (ints) to pid_files (paths) - - """ - graceful = kwargs.get('graceful') - if graceful and self.server in GRACEFUL_SHUTDOWN_SERVERS: - sig = signal.SIGHUP - else: - sig = signal.SIGTERM - return self.signal_pids(sig, **kwargs) - - def status(self, pids=None, **kwargs): - """Display status of server - - :param: pids, if not supplied pids will be populated automatically - :param: number, if supplied will only lookup the nth server - - :returns: 1 if server is not running, 0 otherwise - """ - if pids is None: - pids = self.get_running_pids(**kwargs) - if not pids: - number = kwargs.get('number', 0) - if number: - kwargs['quiet'] = True - ini_files = self.ini_files(**kwargs) - if ini_files: - print "%s #%d not running (%s)" % (self.server, number, - ini_files[0]) - else: - print "No %s running" % self.server - return 1 - for pid, pid_file in pids.items(): - ini_file = self.get_ini_file_name(pid_file) - print "%s running (%s - %s)" % (self.server, pid, ini_file) - return 0 - - def spawn(self, ini_file, once=False, wait=False, daemon=True, **kwargs): - """Launch a subprocess for this server. - - :param ini_file: path to ini_file to use as first arg - :param once: boolean, add once argument to command - :param wait: boolean, if true capture stdout with a pipe - :param daemon: boolean, if true ask server to log to console - - :returns : the pid of the spawned process - """ - args = [self.cmd, ini_file] - if once: - args.append('once') - if not daemon: - # ask the server to log to console - args.append('verbose') - - # figure out what we're going to do with stdio - if not daemon: - # do nothing, this process is open until the spawns close anyway - re_out = None - re_err = None - else: - re_err = subprocess.STDOUT - if wait: - # we're going to need to block on this... - re_out = subprocess.PIPE - else: - re_out = open(os.devnull, 'w+b') - proc = subprocess.Popen(args, stdout=re_out, stderr=re_err) - pid_file = self.get_pid_file_name(ini_file) - write_file(pid_file, proc.pid) - self.procs.append(proc) - return proc.pid - - def wait(self, **kwargs): - """ - wait on spawned procs to start - """ - status = 0 - for proc in self.procs: - # wait for process to close it's stdout - output = proc.stdout.read() - if output: - print output - if proc.returncode: - status += 1 - return status - - def interact(self, **kwargs): - """ - wait on spawned procs to terminate - """ - status = 0 - for proc in self.procs: - # wait for process to terminate - proc.communicate()[0] - if proc.returncode: - status += 1 - return status - - def launch(self, **kwargs): - """ - Collect ini files and attempt to spawn the processes for this server - """ - ini_files = self.ini_files(**kwargs) - if not ini_files: - return [] - - pids = self.get_running_pids(**kwargs) - - already_started = False - for pid, pid_file in pids.items(): - ini_file = self.get_ini_file_name(pid_file) - # for legacy compat you can't start other servers if one server is - # already running (unless -n specifies which one you want), this - # restriction could potentially be lifted, and launch could start - # any unstarted instances - if ini_file in ini_files: - already_started = True - print "%s running (%s - %s)" % (self.server, pid, ini_file) - elif not kwargs.get('number', 0): - already_started = True - print "%s running (%s - %s)" % (self.server, pid, pid_file) - - if already_started: - print "%s already started..." % self.server - #self.status(pids) - return [] - - if self.server not in START_ONCE_SERVERS: - kwargs['once'] = False - - pids = {} - for ini_file in ini_files: - if kwargs.get('once'): - msg = 'Running %s once' % self.server - else: - msg = 'Starting %s' % self.server - print '%s...(%s)' % (msg, ini_file) - pid = self.spawn(ini_file, **kwargs) - pids[pid] = ini_file - - return pids - - def stop(self, **kwargs): - """Send stop signals to pids for this server - - :returns: a dict mapping pids (ints) to pid_files (paths) - - """ - return self.kill_running_pids(**kwargs) +from swift.common.manager import Server, Manager, UnknownCommandError USAGE = """%prog [ Commands: -""" + '\n'.join(["%16s: %s" % x for x in SwiftInit.list_commands()]) +""" + '\n'.join(["%16s: %s" % x for x in Manager.list_commands()]) def main(): @@ -682,16 +51,16 @@ def main(): command = args[-1] servers = args[:-1] - if len(servers) == 1: - # this is just a stupid swap for me cause I always try to "start main" - commands, docs = zip(*SwiftInit.list_commands()) - if servers[0] in commands: - command, servers = servers[0], [command] + # this is just a silly swap for me cause I always try to "start main" + commands = dict(Manager.list_commands()).keys() + if command not in commands and servers[0] in commands: + servers.append(command) + command = servers.pop(0) - controller = SwiftInit(servers) + manager = Manager(servers) try: - status = controller.run_command(command, **options.__dict__) - except UnknownCommand: + status = manager.run_command(command, **options.__dict__) + except UnknownCommandError: parser.print_help() print 'ERROR: unknown command, %s' % command return 1 diff --git a/swift/common/utils.py b/swift/common/utils.py index cdbca7532a..4c9a3034ee 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -34,6 +34,7 @@ from ConfigParser import ConfigParser, NoSectionError, NoOptionError from optparse import OptionParser from tempfile import mkstemp import cPickle as pickle +import glob import eventlet @@ -715,6 +716,58 @@ def write_pickle(obj, dest, tmp): renamer(tmppath, dest) +def search_tree(root, glob_match, ext): + """Look in root, for any files/dirs matching glob, recurively traversing + any found directories looking for files ending with ext + + :param root: start of search path + :param glob_match: glob to match in root, matching dirs are traversed with + os.walk + :param ext: only files that end in ext will be returned + + :returns: list of full paths to matching files, sorted + + """ + found_files = [] + for path in glob.glob(os.path.join(root, glob_match)): + if path.endswith(ext): + found_files.append(path) + else: + for root, dirs, files in os.walk(path): + for file in files: + if file.endswith(ext): + found_files.append(os.path.join(root, file)) + return sorted(found_files) + + +def write_file(path, contents): + """Write contents to file at path + + :param path: any path, subdirs will be created as needed + :param contents: data to write to file, will be converted to string + + """ + dir, name = os.path.split(path) + if not os.path.exists(dir): + try: + os.makedirs(dir) + except OSError, err: + if err.errno == errno.EACCES: + sys.exit('Unable to create %s. Running as non-root?' % dir) + with open(path, 'w') as f: + f.write('%s' % contents) + + +def remove_file(path): + """Quiet wrapper for os.unlink, OSErrors are suppressed + + :param path: first and only argument passed to os.unlink + """ + try: + os.unlink(path) + except OSError: + pass + def audit_location_generator(devices, datadir, mount_check=True, logger=None): ''' Given a devices path and a data directory, yield (path, device, diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index a93c21aa8a..fff37279f7 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -116,6 +116,7 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # redirect errors to logger and close stdio capture_stdio(logger) + # bind to address and port sock = get_socket(conf, default_port=kwargs.get('default_port', 8080)) # remaining tasks should not require elevated privileges diff --git a/test/bin/__init__.py b/test/bin/__init__.py deleted file mode 100644 index 4d50c67b05..0000000000 --- a/test/bin/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -'''Tests for packaged "bin" scripts -''' - -import os -import imp # access the import internals - -bin_test_dir = os.path.dirname(__file__) - -bin_dir = os.path.join(bin_test_dir, '../../bin') - -# build up logical mapping of bin 'file-name' to (module, path) -modules = {} -for file_name in os.listdir(bin_dir): - module = '.'.join(['test', 'bin', file_name.replace('-', '_')]) - path = os.path.join(bin_dir, file_name) - modules[file_name] = (module, path) - -def get_bin_module(bin_file_name): - name, path = modules[bin_file_name] - try: - module = imp.load_source(name, path) - finally: - # another option would be adding bin/*c to .bzrignore - try: - os.unlink(path + 'c') - except OSError: - pass - return module - -# this might not be safe on source files that don't test __name__=="__main__" -swift_init = get_bin_module('swift-init') -st = get_bin_module('st') diff --git a/test/bin/test_swift_init.py b/test/bin/test_swift_init.py deleted file mode 100644 index fb623c3ec7..0000000000 --- a/test/bin/test_swift_init.py +++ /dev/null @@ -1,1414 +0,0 @@ -# Copyright (c) 2010 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -from nose import SkipTest - -import os -import sys -import signal -from shutil import rmtree -from contextlib import contextmanager -from tempfile import mkdtemp -from collections import defaultdict -from threading import Thread -from time import sleep - -from test import bin # for reloading... -from test.bin import swift_init # for testing... - -DUMMY_SIG = 1 - - -@contextmanager -def temptree(files, contents=''): - # generate enough contents to fill the files - c = len(files) - contents = (list(contents) + [''] * c)[:c] - tempdir = mkdtemp() - for path, content in zip(files, contents): - if os.path.isabs(path): - path = '.' + path - new_path = os.path.join(tempdir, path) - subdir = os.path.dirname(new_path) - if not os.path.exists(subdir): - os.makedirs(subdir) - with open(new_path, 'w') as f: - f.write(str(content)) - try: - yield tempdir - finally: - rmtree(tempdir) - - -class MockOs(): - - def __init__(self, pids): - self.running_pids = pids - self.pid_sigs = defaultdict(list) - self.closed_fds = [] - self.child_pid = 9999 # fork defaults to test parent process path - self.execlp_called = False - - def kill(self, pid, sig): - if pid not in self.running_pids: - raise OSError(3, 'No such process') - self.pid_sigs[pid].append(sig) - - def __getattr__(self, name): - # I only over-ride portions of the os module - try: - return object.__getattr__(self, name) - except AttributeError: - return getattr(os, name) - - -def pop_stream(f): - """read everything out of file from the top and clear it out - """ - f.flush() - f.seek(0) - output = f.read() - f.seek(0) - f.truncate() - #print >> sys.stderr, output - return output - - -class TestSwiftInitModule(unittest.TestCase): - - def test_servers(self): - main_plus_rest = set(swift_init.MAIN_SERVERS + swift_init.REST_SERVERS) - self.assertEquals(set(swift_init.ALL_SERVERS), main_plus_rest) - # make sure there's no server listed in both - self.assertEquals(len(main_plus_rest), len(swift_init.MAIN_SERVERS) + - len(swift_init.REST_SERVERS)) - - def test_setup_env(self): - # TODO: tests - raise SkipTest - - def test_search_tree(self): - # file match & ext miss - with temptree(['asdf.conf', 'blarg.conf', 'asdf.cfg']) as t: - asdf = swift_init.search_tree(t, 'a*', '.conf') - self.assertEquals(len(asdf), 1) - self.assertEquals(asdf[0], - os.path.join(t, 'asdf.conf')) - - # multi-file match & glob miss & sort - with temptree(['application.bin', 'apple.bin', 'apropos.bin']) as t: - app_bins = swift_init.search_tree(t, 'app*', 'bin') - self.assertEquals(len(app_bins), 2) - self.assertEquals(app_bins[0], - os.path.join(t, 'apple.bin')) - self.assertEquals(app_bins[1], - os.path.join(t, 'application.bin')) - - # test file in folder & ext miss & glob miss - files = ( - 'sub/file1.ini', - 'sub/file2.conf', - 'sub.bin', - 'bus.ini', - 'bus/file3.ini', - ) - with temptree(files) as t: - sub_ini = swift_init.search_tree(t, 'sub*', '.ini') - self.assertEquals(len(sub_ini), 1) - self.assertEquals(sub_ini[0], - os.path.join(t, 'sub/file1.ini')) - - # test multi-file in folder & sub-folder & ext miss & glob miss - files = ( - 'folder_file.txt', - 'folder/1.txt', - 'folder/sub/2.txt', - 'folder2/3.txt', - 'Folder3/4.txt' - 'folder.rc', - ) - with temptree(files) as t: - folder_texts = swift_init.search_tree(t, 'folder*', '.txt') - self.assertEquals(len(folder_texts), 4) - f1 = os.path.join(t, 'folder_file.txt') - f2 = os.path.join(t, 'folder/1.txt') - f3 = os.path.join(t, 'folder/sub/2.txt') - f4 = os.path.join(t, 'folder2/3.txt') - for f in [f1, f2, f3, f4]: - self.assert_(f in folder_texts) - - def test_write_file(self): - with temptree([]) as t: - file_name = os.path.join(t, 'test') - swift_init.write_file(file_name, 'test') - with open(file_name, 'r') as f: - contents = f.read() - self.assertEquals(contents, 'test') - # and also subdirs - file_name = os.path.join(t, 'subdir/test2') - swift_init.write_file(file_name, 'test2') - with open(file_name, 'r') as f: - contents = f.read() - self.assertEquals(contents, 'test2') - # but can't over-write files - file_name = os.path.join(t, 'subdir/test2/test3') - self.assertRaises(IOError, swift_init.write_file, file_name, - 'test3') - - def test_remove_file(self): - with temptree([]) as t: - file_name = os.path.join(t, 'blah.pid') - # assert no raise - self.assertEquals(os.path.exists(file_name), False) - self.assertEquals(swift_init.remove_file(file_name), None) - with open(file_name, 'w') as f: - f.write('1') - self.assert_(os.path.exists(file_name)) - self.assertEquals(swift_init.remove_file(file_name), None) - self.assertFalse(os.path.exists(file_name)) - - def test_command_wrapper(self): - @swift_init.command - def myfunc(arg1): - """test doc - """ - return arg1 - - self.assertEquals(myfunc.__doc__.strip(), 'test doc') - self.assertEquals(myfunc(1), 1) - self.assertEquals(myfunc(0), 0) - self.assertEquals(myfunc(True), 1) - self.assertEquals(myfunc(False), 0) - self.assert_(hasattr(myfunc, 'publicly_accessible')) - self.assert_(myfunc.publicly_accessible) - - def test_exc(self): - self.assert_(issubclass(swift_init.UnknownCommand, Exception)) - - -class TestSwiftServerClass(unittest.TestCase): - - def tearDown(self): - reload(bin) - - def join_swift_dir(self, path): - return os.path.join(swift_init.SWIFT_DIR, path) - - def join_run_dir(self, path): - return os.path.join(swift_init.RUN_DIR, path) - - def test_create_server(self): - server = swift_init.SwiftServer('proxy') - self.assertEquals(server.server, 'proxy-server') - self.assertEquals(server.type, 'proxy') - self.assertEquals(server.cmd, 'swift-proxy-server') - server = swift_init.SwiftServer('object-replicator') - self.assertEquals(server.server, 'object-replicator') - self.assertEquals(server.type, 'object') - self.assertEquals(server.cmd, 'swift-object-replicator') - - def test_server_to_string(self): - server = swift_init.SwiftServer('Proxy') - self.assertEquals(str(server), 'proxy-server') - server = swift_init.SwiftServer('object-replicator') - self.assertEquals(str(server), 'object-replicator') - - def test_server_repr(self): - server = swift_init.SwiftServer('proxy') - self.assert_(server.__class__.__name__ in repr(server)) - self.assert_(str(server) in repr(server)) - - def test_server_equality(self): - server1 = swift_init.SwiftServer('Proxy') - server2 = swift_init.SwiftServer('proxy-server') - self.assertEquals(server1, server2) - # it is NOT a string - self.assertNotEquals(server1, 'proxy-server') - - def test_get_pid_file_name(self): - server = swift_init.SwiftServer('proxy') - ini_file = self.join_swift_dir('proxy-server.conf') - pid_file = self.join_run_dir('proxy-server.pid') - self.assertEquals(pid_file, server.get_pid_file_name(ini_file)) - server = swift_init.SwiftServer('object-replicator') - ini_file = self.join_swift_dir('object-server/1.conf') - pid_file = self.join_run_dir('object-replicator/1.pid') - self.assertEquals(pid_file, server.get_pid_file_name(ini_file)) - server = swift_init.SwiftServer('container-auditor') - ini_file = self.join_swift_dir( - 'container-server/1/container-auditor.conf') - pid_file = self.join_run_dir( - 'container-auditor/1/container-auditor.pid') - self.assertEquals(pid_file, server.get_pid_file_name(ini_file)) - - def test_get_ini_file_name(self): - server = swift_init.SwiftServer('proxy') - ini_file = self.join_swift_dir('proxy-server.conf') - pid_file = self.join_run_dir('proxy-server.pid') - self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) - server = swift_init.SwiftServer('object-replicator') - ini_file = self.join_swift_dir('object-server/1.conf') - pid_file = self.join_run_dir('object-replicator/1.pid') - self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) - server = swift_init.SwiftServer('container-auditor') - ini_file = self.join_swift_dir( - 'container-server/1/container-auditor.conf') - pid_file = self.join_run_dir( - 'container-auditor/1/container-auditor.pid') - self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) - - def test_ini_files(self): - # test get single ini file - ini_files = ( - 'proxy-server.conf', - 'proxy-server.ini', - 'auth-server.conf', - ) - with temptree(ini_files) as t: - swift_init.SWIFT_DIR = t - server = swift_init.SwiftServer('proxy') - ini_files = server.ini_files() - self.assertEquals(len(ini_files), 1) - ini_file = ini_files[0] - proxy_conf = self.join_swift_dir('proxy-server.conf') - self.assertEquals(ini_file, proxy_conf) - - # test multi server conf files & grouping of server-type config - ini_files = ( - 'object-server1.conf', - 'object-server/2.conf', - 'object-server/object3.conf', - 'object-server/conf/server4.conf', - 'object-server.txt', - 'proxy-server.conf', - ) - with temptree(ini_files) as t: - swift_init.SWIFT_DIR = t - server = swift_init.SwiftServer('object-replicator') - ini_files = server.ini_files() - self.assertEquals(len(ini_files), 4) - c1 = self.join_swift_dir('object-server1.conf') - c2 = self.join_swift_dir('object-server/2.conf') - c3 = self.join_swift_dir('object-server/object3.conf') - c4 = self.join_swift_dir('object-server/conf/server4.conf') - for c in [c1, c2, c3, c4]: - self.assert_(c in ini_files) - # test configs returned sorted - sorted_confs = sorted([c1, c2, c3, c4]) - self.assertEquals(ini_files, sorted_confs) - - # test get single numbered conf - ini_files = ( - 'account-server/1.conf', - 'account-server/2.conf', - 'account-server/3.conf', - 'account-server/4.conf', - ) - with temptree(ini_files) as t: - swift_init.SWIFT_DIR = t - server = swift_init.SwiftServer('account') - ini_files = server.ini_files(number=2) - self.assertEquals(len(ini_files), 1) - ini_file = ini_files[0] - self.assertEquals(ini_file, - self.join_swift_dir('account-server/2.conf')) - # test missing config number - ini_files = server.ini_files(number=5) - self.assertFalse(ini_files) - - # test verbose & quiet - ini_files = ( - 'auth-server.ini', - 'container-server/1.conf', - ) - with temptree(ini_files) as t: - swift_init.SWIFT_DIR = t - old_stdout = sys.stdout - try: - with open(os.path.join(t, 'output'), 'w+') as f: - sys.stdout = f - server = swift_init.SwiftServer('auth') - # check warn "unable to locate" - ini_files = server.ini_files() - self.assertFalse(ini_files) - self.assert_('unable to locate' in pop_stream(f).lower()) - # check quiet will silence warning - ini_files = server.ini_files(verbose=True, quiet=True) - self.assertEquals(pop_stream(f), '') - # check found config no warning - server = swift_init.SwiftServer('container-auditor') - ini_files = server.ini_files() - self.assertEquals(pop_stream(f), '') - # check missing config number warn "unable to locate" - ini_files = server.ini_files(number=2) - self.assert_('unable to locate' in pop_stream(f).lower()) - # check verbose lists configs - ini_files = server.ini_files(number=2, verbose=True) - c1 = self.join_swift_dir('container-server/1.conf') - self.assert_(c1 in pop_stream(f)) - finally: - sys.stdout = old_stdout - - def test_iter_pid_files(self): - """ - SwiftServer.iter_pid_files is kinda boring, test the - SwiftServer.pid_files stuff here as well - """ - pid_files = ( - ('proxy-server.pid', 1), - ('auth-server.pid', 'blah'), - ('object-replicator/1.pid', 11), - ('object-replicator/2.pid', 12), - ) - files, contents = zip(*pid_files) - with temptree(files, contents) as t: - swift_init.RUN_DIR = t - server = swift_init.SwiftServer('proxy') - # test get one file - iter = server.iter_pid_files() - pid_file, pid = iter.next() - self.assertEquals(pid_file, self.join_run_dir('proxy-server.pid')) - self.assertEquals(pid, 1) - # ... and only one file - self.assertRaises(StopIteration, iter.next) - # test invalid value in pid file - server = swift_init.SwiftServer('auth') - self.assertRaises(ValueError, server.iter_pid_files().next) - # test object-server doesn't steal pids from object-replicator - server = swift_init.SwiftServer('object') - self.assertRaises(StopIteration, server.iter_pid_files().next) - # test multi-pid iter - server = swift_init.SwiftServer('object-replicator') - real_map = { - 11: self.join_run_dir('object-replicator/1.pid'), - 12: self.join_run_dir('object-replicator/2.pid'), - } - pid_map = {} - for pid_file, pid in server.iter_pid_files(): - pid_map[pid] = pid_file - self.assertEquals(pid_map, real_map) - - # test get pid_files by number - ini_files = ( - 'object-server/1.conf', - 'object-server/2.conf', - 'object-server/3.conf', - 'object-server/4.conf', - ) - - pid_files = ( - ('object-server/1.pid', 1), - ('object-server/2.pid', 2), - ('object-server/5.pid', 5), - ) - - with temptree(ini_files) as swift_dir: - swift_init.SWIFT_DIR = swift_dir - files, pids = zip(*pid_files) - with temptree(files, pids) as t: - swift_init.RUN_DIR = t - server = swift_init.SwiftServer('object') - # test get all pid files - real_map = { - 1: self.join_run_dir('object-server/1.pid'), - 2: self.join_run_dir('object-server/2.pid'), - 5: self.join_run_dir('object-server/5.pid'), - } - pid_map = {} - for pid_file, pid in server.iter_pid_files(): - pid_map[pid] = pid_file - self.assertEquals(pid_map, real_map) - # test get pid with matching conf - pids = list(server.iter_pid_files(number=2)) - self.assertEquals(len(pids), 1) - pid_file, pid = pids[0] - self.assertEquals(pid, 2) - pid_two = self.join_run_dir('object-server/2.pid') - self.assertEquals(pid_file, pid_two) - # try to iter on a pid number with a matching conf but no pid - pids = list(server.iter_pid_files(number=3)) - self.assertFalse(pids) - # test get pids w/o matching conf - pids = list(server.iter_pid_files(number=5)) - self.assertFalse(pids) - - def test_signal_pids(self): - pid_files = ( - ('proxy-server.pid', 1), - ('auth-server.pid', 2), - ) - files, pids = zip(*pid_files) - with temptree(files, pids) as t: - swift_init.RUN_DIR = t - # mock os with both pids running - swift_init.os = MockOs([1, 2]) - server = swift_init.SwiftServer('proxy') - pids = server.signal_pids(DUMMY_SIG) - self.assertEquals(len(pids), 1) - self.assert_(1 in pids) - self.assertEquals(swift_init.os.pid_sigs[1], [DUMMY_SIG]) - # make sure other process not signaled - self.assertFalse(2 in pids) - self.assertFalse(2 in swift_init.os.pid_sigs) - # capture stdio - old_stdout = sys.stdout - try: - with open(os.path.join(t, 'output'), 'w+') as f: - sys.stdout = f - #test print details - pids = server.signal_pids(DUMMY_SIG) - output = pop_stream(f) - self.assert_('pid: %s' % 1 in output) - self.assert_('signal: %s' % DUMMY_SIG in output) - # test no details on signal.SIG_DFL - pids = server.signal_pids(signal.SIG_DFL) - self.assertEquals(pop_stream(f), '') - # reset mock os so only the other server is running - swift_init.os = MockOs([2]) - # test pid not running - pids = server.signal_pids(signal.SIG_DFL) - self.assert_(1 not in pids) - self.assert_(1 not in swift_init.os.pid_sigs) - # test remove stale pid file - self.assertFalse(os.path.exists( - self.join_run_dir('proxy-server.pid'))) - # reset mock os with no running pids - swift_init.os = MockOs([]) - server = swift_init.SwiftServer('auth') - # test verbose warns on removing pid file - pids = server.signal_pids(signal.SIG_DFL, verbose=True) - output = pop_stream(f) - self.assert_('stale pid' in output.lower()) - auth_pid = self.join_run_dir('auth-server.pid') - self.assert_(auth_pid in output) - finally: - sys.stdout = old_stdout - - def test_get_running_pids(self): - # test only gets running pids - pid_files = ( - ('test-server1.pid', 1), - ('test-server2.pid', 2), - ) - with temptree(*zip(*pid_files)) as t: - swift_init.RUN_DIR = t - server = swift_init.SwiftServer('test-server') - # mock os, only pid '1' is running - swift_init.os = MockOs([1]) - running_pids = server.get_running_pids() - self.assertEquals(len(running_pids), 1) - self.assert_(1 in running_pids) - self.assert_(2 not in running_pids) - # test persistant running pid files - self.assert_(os.path.exists(os.path.join(t, 'test-server1.pid'))) - # test clean up stale pids - pid_two = self.join_swift_dir('test-server2.pid') - self.assertFalse(os.path.exists(pid_two)) - # reset mock os, no pids running - swift_init.os = MockOs([]) - running_pids = server.get_running_pids() - self.assertFalse(running_pids) - # and now all pid files are cleaned out - pid_one = self.join_run_dir('test-server1.pid') - self.assertFalse(os.path.exists(pid_one)) - all_pids = os.listdir(t) - self.assertEquals(len(all_pids), 0) - - # test only get pids for right server - pid_files = ( - ('thing-doer.pid', 1), - ('thing-sayer.pid', 2), - ('other-doer.pid', 3), - ('other-sayer.pid', 4), - ) - files, pids = zip(*pid_files) - with temptree(files, pids) as t: - swift_init.RUN_DIR = t - # all pids are running - swift_init.os = MockOs(pids) - server = swift_init.SwiftServer('thing-doer') - running_pids = server.get_running_pids() - # only thing-doer.pid, 1 - self.assertEquals(len(running_pids), 1) - self.assert_(1 in running_pids) - # no other pids returned - for n in (2, 3, 4): - self.assert_(n not in running_pids) - # assert stale pids for other servers ignored - swift_init.os = MockOs([1]) # only thing-doer is running - running_pids = server.get_running_pids() - for f in ('thing-sayer.pid', 'other-doer.pid', 'other-sayer.pid'): - # other server pid files persist - self.assert_(os.path.exists, os.path.join(t, f)) - # verify that servers are in fact not running - for server_name in ('thing-sayer', 'other-doer', 'other-sayer'): - server = swift_init.SwiftServer(server_name) - running_pids = server.get_running_pids() - self.assertFalse(running_pids) - # and now all OTHER pid files are cleaned out - all_pids = os.listdir(t) - self.assertEquals(len(all_pids), 1) - self.assert_(os.path.exists(os.path.join(t, 'thing-doer.pid'))) - - def test_kill_running_pids(self): - pid_files = ( - ('object-server.pid', 1), - ('object-replicator1.pid', 11), - ('object-replicator2.pid', 12), - ) - files, pids = zip(*pid_files) - with temptree(files, pids) as t: - swift_init.RUN_DIR = t - server = swift_init.SwiftServer('object') - # test no servers running - pids = server.kill_running_pids() - self.assertFalse(pids) - # start up pid - swift_init.os = MockOs([1]) - # test kill one pid - pids = server.kill_running_pids() - self.assertEquals(len(pids), 1) - self.assert_(1 in pids) - self.assertEquals(swift_init.os.pid_sigs[1], [signal.SIGTERM]) - # reset os mock - swift_init.os = MockOs([1]) - # test shutdown - self.assert_('object-server' in - swift_init.GRACEFUL_SHUTDOWN_SERVERS) - pids = server.kill_running_pids(graceful=True) - self.assertEquals(len(pids), 1) - self.assert_(1 in pids) - self.assertEquals(swift_init.os.pid_sigs[1], [signal.SIGHUP]) - # start up other servers - swift_init.os = MockOs([11, 12]) - # test multi server kill & ignore graceful on unsupport server - self.assertFalse('object-replicator' in - swift_init.GRACEFUL_SHUTDOWN_SERVERS) - server = swift_init.SwiftServer('object-replicator') - pids = server.kill_running_pids(graceful=True) - self.assertEquals(len(pids), 2) - for pid in (11, 12): - self.assert_(pid in pids) - self.assertEquals(swift_init.os.pid_sigs[pid], - [signal.SIGTERM]) - # and the other pid is of course not signaled - self.assert_(1 not in swift_init.os.pid_sigs) - - def test_status(self): - ini_files = ( - 'test-server/1.conf', - 'test-server/2.conf', - 'test-server/3.conf', - 'test-server/4.conf', - ) - - pid_files = ( - ('test-server/1.pid', 1), - ('test-server/2.pid', 2), - ('test-server/3.pid', 3), - ('test-server/4.pid', 4), - ) - - with temptree(ini_files) as swift_dir: - swift_init.SWIFT_DIR = swift_dir - files, pids = zip(*pid_files) - with temptree(files, pids) as t: - swift_init.RUN_DIR = t - # setup running servers - server = swift_init.SwiftServer('test') - # capture stdio - old_stdout = sys.stdout - try: - with open(os.path.join(t, 'output'), 'w+') as f: - sys.stdout = f - # test status for all running - swift_init.os = MockOs(pids) - self.assertEquals(server.status(), 0) - output = pop_stream(f).strip().splitlines() - self.assertEquals(len(output), 4) - for line in output: - self.assert_('test-server running' in line) - # test get single server by number - self.assertEquals(server.status(number=4), 0) - output = pop_stream(f).strip().splitlines() - self.assertEquals(len(output), 1) - line = output[0] - self.assert_('test-server running' in line) - conf_four = self.join_swift_dir(ini_files[3]) - self.assert_('4 - %s' % conf_four in line) - # test some servers not running - swift_init.os = MockOs([1, 2, 3]) - self.assertEquals(server.status(), 0) - output = pop_stream(f).strip().splitlines() - self.assertEquals(len(output), 3) - for line in output: - self.assert_('test-server running' in line) - # test single server not running - swift_init.os = MockOs([1, 2]) - self.assertEquals(server.status(number=3), 1) - output = pop_stream(f).strip().splitlines() - self.assertEquals(len(output), 1) - line = output[0] - self.assert_('not running' in line) - conf_three = self.join_swift_dir(ini_files[2]) - self.assert_(conf_three in line) - # test no running pids - swift_init.os = MockOs([]) - self.assertEquals(server.status(), 1) - output = pop_stream(f).lower() - self.assert_('no test-server running' in output) - # test use provided pids - pids = { - 1: '1.pid', - 2: '2.pid', - } - # shouldn't call get_running_pids - called = [] - - def mock(*args, **kwargs): - called.append(True) - server.get_running_pids = mock - status = server.status(pids=pids) - self.assertEquals(status, 0) - self.assertFalse(called) - output = pop_stream(f).strip().splitlines() - self.assertEquals(len(output), 2) - for line in output: - self.assert_('test-server running' in line) - finally: - sys.stdout = old_stdout - - def test_spawn(self): - - # mocks - class MockProcess(): - - NOTHING = 'default besides None' - STDOUT = 'stdout' - PIPE = 'pipe' - - def __init__(self, pids=None): - if pids is None: - pids = [] - self.pids = (p for p in pids) - - def Popen(self, args, **kwargs): - return MockProc(self.pids.next(), args, **kwargs) - - class MockProc(): - - def __init__(self, pid, args, stdout=MockProcess.NOTHING, - stderr=MockProcess.NOTHING): - self.pid = pid - self.args = args - self.stdout = stdout - if stderr == MockProcess.STDOUT: - self.stderr = self.stdout - else: - self.stderr = stderr - - # setup running servers - server = swift_init.SwiftServer('test') - - with temptree(['test-server.conf']) as swift_dir: - swift_init.SWIFT_DIR = swift_dir - with temptree([]) as t: - swift_init.RUN_DIR = t - old_subprocess = swift_init.subprocess - try: - # test single server process calls spawn once - swift_init.subprocess = MockProcess([1]) - conf_file = self.join_swift_dir('test-server.conf') - # spawn server no kwargs - server.spawn(conf_file) - # test pid file - pid_file = self.join_run_dir('test-server.pid') - self.assert_(os.path.exists(pid_file)) - pid_on_disk = int(open(pid_file).read().strip()) - self.assertEquals(pid_on_disk, 1) - # assert procs args - self.assert_(server.procs) - self.assertEquals(len(server.procs), 1) - proc = server.procs[0] - expected_args = [ - 'swift-test-server', - conf_file, - ] - self.assertEquals(proc.args, expected_args) - # assert stdout is /dev/null - self.assert_(isinstance(proc.stdout, file)) - self.assertEquals(proc.stdout.name, os.devnull) - self.assertEquals(proc.stdout.mode, 'w+b') - self.assertEquals(proc.stderr, proc.stdout) - # test multi server process calls spawn multiple times - swift_init.subprocess = MockProcess([11, 12, 13, 14]) - conf1 = self.join_swift_dir('test-server/1.conf') - conf2 = self.join_swift_dir('test-server/2.conf') - conf3 = self.join_swift_dir('test-server/3.conf') - conf4 = self.join_swift_dir('test-server/4.conf') - server = swift_init.SwiftServer('test') - # test server run once - server.spawn(conf1, once=True) - self.assert_(server.procs) - self.assertEquals(len(server.procs), 1) - proc = server.procs[0] - expected_args = ['swift-test-server', conf1, 'once'] - self.assertEquals(proc.args, expected_args) - # assert stdout is /dev/null - self.assert_(isinstance(proc.stdout, file)) - self.assertEquals(proc.stdout.name, os.devnull) - self.assertEquals(proc.stdout.mode, 'w+b') - self.assertEquals(proc.stderr, proc.stdout) - # test server not daemon - server.spawn(conf2, daemon=False) - self.assert_(server.procs) - self.assertEquals(len(server.procs), 2) - proc = server.procs[1] - expected_args = ['swift-test-server', conf2, 'verbose'] - self.assertEquals(proc.args, expected_args) - # assert stdout is not changed - self.assertEquals(proc.stdout, None) - self.assertEquals(proc.stderr, None) - # test server wait - server.spawn(conf3, wait=True) - self.assert_(server.procs) - self.assertEquals(len(server.procs), 3) - proc = server.procs[2] - # assert stdout is piped - self.assertEquals(proc.stdout, MockProcess.PIPE) - self.assertEquals(proc.stderr, proc.stdout) - # test not daemon over-rides wait - server.spawn(conf4, wait=True, daemon=False, once=True) - self.assert_(server.procs) - self.assertEquals(len(server.procs), 4) - proc = server.procs[3] - expected_args = ['swift-test-server', conf4, 'once', - 'verbose'] - self.assertEquals(proc.args, expected_args) - # daemon behavior should trump wait, once shouldn't matter - self.assertEquals(proc.stdout, None) - self.assertEquals(proc.stderr, None) - # assert pids - for i, proc in enumerate(server.procs): - pid_file = self.join_run_dir('test-server/%d.pid' % - (i + 1)) - pid_on_disk = int(open(pid_file).read().strip()) - self.assertEquals(pid_on_disk, proc.pid) - finally: - swift_init.subprocess = old_subprocess - - def test_wait(self): - server = swift_init.SwiftServer('test') - self.assertEquals(server.wait(), 0) - - class MockProcess(Thread): - def __init__(self, delay=0.1, fail_to_start=False): - Thread.__init__(self) - # setup pipe - rfd, wfd = os.pipe() - # subprocess connection to read stdout - self.stdout = os.fdopen(rfd) - # real process connection to write stdout - self._stdout = os.fdopen(wfd, 'w') - self.delay = delay - self.finished = False - self.returncode = None - if fail_to_start: - self.run = self.fail - - def __enter__(self): - self.start() - return self - - def __exit__(self, *args): - if self.isAlive(): - self.join() - - def close_stdout(self): - self._stdout.flush() - with open(os.devnull, 'wb') as nullfile: - try: - os.dup2(nullfile.fileno(), self._stdout.fileno()) - except OSError: - pass - - def fail(self): - print >>self._stdout, 'mock process started' - sleep(self.delay) # perform setup processing - print >>self._stdout, 'mock process failed to start' - self.returncode = 1 - self.close_stdout() - - def run(self): - print >>self._stdout, 'mock process started' - sleep(self.delay) # perform setup processing - print >>self._stdout, 'setup complete!' - self.close_stdout() - sleep(self.delay) # do some more processing - print >>self._stdout, 'mock process finished' - self.finished = True - - with temptree([]) as t: - old_stdout = sys.stdout - try: - with open(os.path.join(t, 'output'), 'w+') as f: - # acctually capture the read stdout (for prints) - sys.stdout = f - # test closing pipe in subprocess unblocks read - with MockProcess() as proc: - server.procs = [proc] - status = server.wait() - self.assertEquals(status, 0) - # wait should return as soon as stdout is closed - self.assert_(proc.isAlive()) - self.assertFalse(proc.finished) - self.assert_(proc.finished) # make sure it did finish... - # test output kwarg prints subprocess output - with MockProcess() as proc: - server.procs = [proc] - status = server.wait(output=True) - output = pop_stream(f) - self.assert_('mock process started' in output) - self.assert_('setup complete' in output) - # make sure we don't get prints after stdout was closed - self.assert_('mock process finished' not in output) - # test process which fails to start - with MockProcess(fail_to_start=True) as proc: - server.procs = [proc] - status = server.wait() - self.assertEquals(status, 1) - self.assert_('failed' in pop_stream(f)) - # test multiple procs - procs = [MockProcess() for i in range(3)] - for proc in procs: - proc.start() - server.procs = procs - status = server.wait() - self.assertEquals(status, 0) - for proc in procs: - self.assert_(proc.isAlive()) - for proc in procs: - proc.join() - finally: - sys.stdout = old_stdout - - def test_interact(self): - class MockProcess(): - - def __init__(self, fail=False): - self.returncode = None - if fail: - self._returncode = 1 - else: - self._returncode = 0 - - def communicate(self): - self.returncode = self._returncode - return '', '' - - server = swift_init.SwiftServer('test') - server.procs = [MockProcess()] - self.assertEquals(server.interact(), 0) - server.procs = [MockProcess(fail=True)] - self.assertEquals(server.interact(), 1) - procs = [] - for fail in (False, True, True): - procs.append(MockProcess(fail=fail)) - server.procs = procs - self.assert_(server.interact() > 0) - - def test_launch(self): - # stubs - ini_files = ( - 'proxy-server.conf', - 'object-server/1.conf', - 'object-server/2.conf', - 'object-server/3.conf', - 'object-server/4.conf', - ) - pid_files = ( - ('proxy-server.pid', 1), - ('proxy-server/2.pid', 2), - ) - - #mocks - class MockSpawn(): - - def __init__(self, pids=None): - self.ini_files = [] - self.kwargs = [] - if not pids: - def one_forever(): - while True: - yield 1 - self.pids = one_forever() - else: - self.pids = (x for x in pids) - - def __call__(self, ini_file, **kwargs): - self.ini_files.append(ini_file) - self.kwargs.append(kwargs) - return self.pids.next() - - with temptree(ini_files) as swift_dir: - swift_init.SWIFT_DIR = swift_dir - files, pids = zip(*pid_files) - with temptree(files, pids) as t: - swift_init.RUN_DIR = t - old_stdout = sys.stdout - try: - with open(os.path.join(t, 'output'), 'w+') as f: - sys.stdout = f - # can't start server w/o an conf - server = swift_init.SwiftServer('test') - self.assertFalse(server.launch()) - # start mock os running all pids - swift_init.os = MockOs(pids) - server = swift_init.SwiftServer('proxy') - # can't start server if it's already running - self.assertFalse(server.launch()) - output = pop_stream(f) - self.assert_('running' in output) - ini_file = self.join_swift_dir('proxy-server.conf') - self.assert_(ini_file in output) - pid_file = self.join_run_dir('proxy-server/2.pid') - self.assert_(pid_file in output) - self.assert_('already started' in output) - # no running pids - swift_init.os = MockOs([]) - # test ignore once for non-start-once server - mock_spawn = MockSpawn([1]) - server.spawn = mock_spawn - ini_file = self.join_swift_dir('proxy-server.conf') - expected = { - 1: ini_file, - } - self.assertEquals(server.launch(once=True), expected) - self.assertEquals(mock_spawn.ini_files, [ini_file]) - expected = { - 'once': False, - } - self.assertEquals(mock_spawn.kwargs, [expected]) - output = pop_stream(f) - self.assert_('Starting' in output) - self.assert_('once' not in output) - # test multi-server kwarg once - server = swift_init.SwiftServer('object-replicator') - mock_spawn = MockSpawn([1, 2, 3, 4]) - server.spawn = mock_spawn - conf1 = self.join_swift_dir('object-server/1.conf') - conf2 = self.join_swift_dir('object-server/2.conf') - conf3 = self.join_swift_dir('object-server/3.conf') - conf4 = self.join_swift_dir('object-server/4.conf') - expected = { - 1: conf1, - 2: conf2, - 3: conf3, - 4: conf4, - } - self.assertEquals(server.launch(once=True), expected) - self.assertEquals(mock_spawn.ini_files, [conf1, conf2, - conf3, conf4]) - expected = { - 'once': True, - } - self.assertEquals(len(mock_spawn.kwargs), 4) - for kwargs in mock_spawn.kwargs: - self.assertEquals(kwargs, expected) - # test number kwarg - mock_spawn = MockSpawn([4]) - server.spawn = mock_spawn - expected = { - 4: conf4, - } - self.assertEquals(server.launch(number=4), expected) - self.assertEquals(mock_spawn.ini_files, [conf4]) - expected = { - 'number': 4 - } - self.assertEquals(mock_spawn.kwargs, [expected]) - - - finally: - sys.stdout = old_stdout - - def test_stop(self): - ini_files = ( - 'account-server/1.conf', - 'account-server/2.conf', - 'account-server/3.conf', - 'account-server/4.conf', - ) - pid_files = ( - ('account-reaper/1.pid', 1), - ('account-reaper/2.pid', 2), - ('account-reaper/3.pid', 3), - ('account-reaper/4.pid', 4), - ) - - with temptree(ini_files) as swift_dir: - swift_init.SWIFT_DIR = swift_dir - files, pids = zip(*pid_files) - with temptree(files, pids) as t: - swift_init.RUN_DIR = t - # start all pids in mock os - swift_init.os = MockOs(pids) - server = swift_init.SwiftServer('account-reaper') - # test kill all running pids - pids = server.stop() - self.assertEquals(len(pids), 4) - for pid in (1, 2, 3, 4): - self.assert_(pid in pids) - self.assertEquals(swift_init.os.pid_sigs[pid], - [signal.SIGTERM]) - conf1 = self.join_swift_dir('account-reaper/1.conf') - conf2 = self.join_swift_dir('account-reaper/2.conf') - conf3 = self.join_swift_dir('account-reaper/3.conf') - conf4 = self.join_swift_dir('account-reaper/4.conf') - # reset mock os with only 2 running pids - swift_init.os = MockOs([3, 4]) - pids = server.stop() - self.assertEquals(len(pids), 2) - for pid in (3, 4): - self.assert_(pid in pids) - self.assertEquals(swift_init.os.pid_sigs[pid], - [signal.SIGTERM]) - self.assertFalse(os.path.exists(conf1)) - self.assertFalse(os.path.exists(conf2)) - # test number kwarg - swift_init.os = MockOs([3, 4]) - pids = server.stop(number=3) - self.assertEquals(len(pids), 1) - expected = { - 3: conf3, - } - self.assert_(pids, expected) - self.assertEquals(swift_init.os.pid_sigs[3], [signal.SIGTERM]) - self.assertFalse(os.path.exists(conf4)) - self.assertFalse(os.path.exists(conf3)) - -class TestSwiftInitClass(unittest.TestCase): - - def test_create(self): - controller = swift_init.SwiftInit(['test']) - self.assertEquals(len(controller.servers), 1) - server = controller.servers.pop() - self.assert_(isinstance(server, swift_init.SwiftServer)) - self.assertEquals(server.server, 'test-server') - # test multi-server and simple dedupe - servers = ['object-replicator', 'object-auditor', 'object-replicator'] - controller = swift_init.SwiftInit(servers) - self.assertEquals(len(controller.servers), 2) - for server in controller.servers: - self.assert_(server.server in servers) - # test all - controller = swift_init.SwiftInit(['all']) - self.assertEquals(len(controller.servers), len(swift_init.ALL_SERVERS)) - for server in controller.servers: - self.assert_(server.server in swift_init.ALL_SERVERS) - # test main - controller = swift_init.SwiftInit(['main']) - self.assertEquals(len(controller.servers), len(swift_init.MAIN_SERVERS)) - for server in controller.servers: - self.assert_(server.server in swift_init.MAIN_SERVERS) - # test rest - controller = swift_init.SwiftInit(['rest']) - self.assertEquals(len(controller.servers), len(swift_init.REST_SERVERS)) - for server in controller.servers: - self.assert_(server.server in swift_init.REST_SERVERS) - # test main + rest == all - controller = swift_init.SwiftInit(['main', 'rest']) - self.assertEquals(len(controller.servers), len(swift_init.ALL_SERVERS)) - for server in controller.servers: - self.assert_(server.server in swift_init.ALL_SERVERS) - # test dedupe - controller = swift_init.SwiftInit(['main', 'rest', 'proxy', 'object', - 'container', 'account']) - self.assertEquals(len(controller.servers), len(swift_init.ALL_SERVERS)) - for server in controller.servers: - self.assert_(server.server in swift_init.ALL_SERVERS) - - #TODO: more tests - def test_watch_server_pids(self): - raise SkipTest - - def test_get_command(self): - raise SkipTest - - def test_list_commands(self): - for cmd, help in swift_init.SwiftInit.list_commands(): - method = getattr(swift_init.SwiftInit, cmd.replace('-', '_'), None) - self.assert_(method, '%s is not a command' % cmd) - self.assert_(getattr(method, 'publicly_accessible', False)) - self.assertEquals(method.__doc__.strip(), help) - - def test_run_command(self): - raise SkipTest - - def test_status(self): - class MockSwiftServer(): - def __init__(self, server): - self.server = server - self.called_kwargs = [] - def status(self, **kwargs): - self.called_kwargs.append(kwargs) - if 'error' in self.server: - return 1 - else: - return 0 - - old_server_class = swift_init.SwiftServer - try: - swift_init.SwiftServer = MockSwiftServer - controller = swift_init.SwiftInit(['test']) - status = controller.status() - self.assertEquals(status, 0) - controller = swift_init.SwiftInit(['error']) - status = controller.status() - self.assertEquals(status, 1) - # test multi-server - controller = swift_init.SwiftInit(['test', 'error']) - kwargs = {'key': 'value'} - status = controller.status(**kwargs) - self.assertEquals(status, 1) - for server in controller.servers: - self.assertEquals(server.called_kwargs, [kwargs]) - finally: - swift_init.SwiftServer = old_server_class - - def test_start(self): - def mock_setup_env(): - getattr(mock_setup_env, 'called', []).append(True) - class MockSwiftServer(): - - def __init__(self, server): - self.server = server - self.called = defaultdict(list) - - def launch(self, **kwargs): - self.called['launch'].append(kwargs) - - def wait(self, **kwargs): - self.called['wait'].append(kwargs) - if 'error' in self.server: - return 1 - else: - return 0 - - def interact(self, **kwargs): - self.called['interact'].append(kwargs) - # TODO: test user quit - """ - if 'raise' in self.server: - raise KeyboardInterrupt - el - """ - if 'error' in self.server: - return 1 - else: - return 0 - - old_setup_env = swift_init.setup_env - old_swift_server = swift_init.SwiftServer - try: - swift_init.setup_env = mock_setup_env - swift_init.SwiftServer = MockSwiftServer - - # test no errors on launch - controller = swift_init.SwiftInit(['proxy', 'error']) - status = controller.start() - self.assertEquals(status, 0) - for server in controller.servers: - self.assertEquals(server.called['launch'], [{}]) - - # test error on wait - controller = swift_init.SwiftInit(['proxy', 'error']) - kwargs = {'wait': True} - status = controller.start(**kwargs) - self.assertEquals(status, 1) - for server in controller.servers: - self.assertEquals(server.called['launch'], [kwargs]) - self.assertEquals(server.called['wait'], [kwargs]) - - # test interact - controller = swift_init.SwiftInit(['proxy', 'error']) - kwargs = {'daemon': False} - status = controller.start(**kwargs) - self.assertEquals(status, 1) - for server in controller.servers: - self.assertEquals(server.called['launch'], [kwargs]) - self.assertEquals(server.called['interact'], [kwargs]) - finally: - swift_init.setup_env = old_setup_env - swift_init.SwiftServer = old_swift_server - - - def test_wait(self): - class MockSwiftServer(): - def __init__(self, server): - self.server = server - self.called = defaultdict(list) - - def launch(self, **kwargs): - self.called['launch'].append(kwargs) - - def wait(self, **kwargs): - self.called['wait'].append(kwargs) - return int('error' in self.server) - - orig_swift_server = swift_init.SwiftServer - try: - swift_init.SwiftServer = MockSwiftServer - # test success - init = swift_init.SwiftInit(['proxy']) - status = init.wait() - self.assertEquals(status, 0) - for server in init.servers: - self.assertEquals(len(server.called['launch']), 1) - called_kwargs = server.called['launch'][0] - self.assert_('wait' in called_kwargs) - self.assert_(called_kwargs['wait']) - self.assertEquals(len(server.called['wait']), 1) - called_kwargs = server.called['wait'][0] - self.assert_('wait' in called_kwargs) - self.assert_(called_kwargs['wait']) - # test error - init = swift_init.SwiftInit(['error']) - status = init.wait() - self.assertEquals(status, 1) - for server in init.servers: - self.assertEquals(len(server.called['launch']), 1) - called_kwargs = server.called['launch'][0] - self.assert_('wait' in called_kwargs) - self.assert_(called_kwargs['wait']) - self.assertEquals(len(server.called['wait']), 1) - called_kwargs = server.called['wait'][0] - self.assert_('wait' in called_kwargs) - self.assert_(called_kwargs['wait']) - # test wait with once option - init = swift_init.SwiftInit(['updater', 'replicator-error']) - status = init.wait(once=True) - self.assertEquals(status, 1) - for server in init.servers: - self.assertEquals(len(server.called['launch']), 1) - called_kwargs = server.called['launch'][0] - self.assert_('wait' in called_kwargs) - self.assert_(called_kwargs['wait']) - self.assert_('once' in called_kwargs) - self.assert_(called_kwargs['once']) - self.assertEquals(len(server.called['wait']), 1) - called_kwargs = server.called['wait'][0] - self.assert_('wait' in called_kwargs) - self.assert_(called_kwargs['wait']) - self.assert_('once' in called_kwargs) - self.assert_(called_kwargs['once']) - finally: - swift_init.SwiftServer = orig_swift_server - - - def test_no_daemon(self): - class MockSwiftServer(): - - def __init__(self, server): - self.server = server - self.called = defaultdict(list) - - def launch(self, **kwargs): - self.called['launch'].append(kwargs) - - def interact(self, **kwargs): - self.called['interact'].append(kwargs) - return int('error' in self.server) - - orig_swift_server = swift_init.SwiftServer - try: - swift_init.SwiftServer = MockSwiftServer - # test success - init = swift_init.SwiftInit(['proxy']) - stats = init.no_daemon() - self.assertEquals(stats, 0) - # test error - init = swift_init.SwiftInit(['proxy', 'object-error']) - stats = init.no_daemon() - self.assertEquals(stats, 1) - # test once - init = swift_init.SwiftInit(['proxy', 'object-error']) - stats = init.no_daemon() - for server in init.servers: - self.assertEquals(len(server.called['launch']), 1) - self.assertEquals(len(server.called['wait']), 0) - self.assertEquals(len(server.called['interact']), 1) - finally: - swift_init.SwiftServer = orig_swift_server - - def test_once(self): - class MockSwiftServer(): - - def __init__(self, server): - self.server = server - self.called = defaultdict(list) - - def launch(self, **kwargs): - return self.called['launch'].append(kwargs) - - - orig_swift_server = swift_init.SwiftServer - try: - swift_init.SwiftServer = MockSwiftServer - # test no errors - init = swift_init.SwiftInit(['account-reaper']) - status = init.once() - self.assertEquals(status, 0) - # test no error code on error - init = swift_init.SwiftInit(['error-reaper']) - status = init.once() - self.assertEquals(status, 0) - for server in init.servers: - self.assertEquals(len(server.called['launch']), 1) - called_kwargs = server.called['launch'][0] - self.assertEquals(called_kwargs, {'once': True}) - self.assertEquals(len(server.called['wait']), 0) - self.assertEquals(len(server.called['interact']), 0) - finally: - swift_init.SwiftServer = orig_swift_server - - - def test_stop(self): - raise SkipTest - - def test_shutdown(self): - raise SkipTest - - def test_restart(self): - raise SkipTest - - def test_reload(self): - raise SkipTest - - def test_force_reload(self): - raise SkipTest - - - -#TODO: test main -class TestMain(unittest.TestCase): - - def test_placeholder(self): - raise SkipTest - - -if __name__ == '__main__': - unittest.main() diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 1895098c2e..c53127e665 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -4,6 +4,8 @@ import os from contextlib import contextmanager from tempfile import NamedTemporaryFile from eventlet.green import socket +from tempfile import mkdtemp +from shutil import rmtree def readuntil2crlfs(fd): @@ -38,6 +40,27 @@ def tmpfile(content): os.unlink(file_name) +@contextmanager +def temptree(files, contents=''): + # generate enough contents to fill the files + c = len(files) + contents = (list(contents) + [''] * c)[:c] + tempdir = mkdtemp() + for path, content in zip(files, contents): + if os.path.isabs(path): + path = '.' + path + new_path = os.path.join(tempdir, path) + subdir = os.path.dirname(new_path) + if not os.path.exists(subdir): + os.makedirs(subdir) + with open(new_path, 'w') as f: + f.write(str(content)) + try: + yield tempdir + finally: + rmtree(tempdir) + + class MockTrue(object): """ Instances of MockTrue evaluate like True diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index c41d147f79..63a1a71cfb 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -16,6 +16,7 @@ """ Tests for swift.common.utils """ from __future__ import with_statement +from test.unit import temptree import logging import mimetools import os @@ -446,5 +447,86 @@ log_name = yarr''' self.assertNotEquals(utils.get_logger.console, old_handler) logger.logger.removeHandler(utils.get_logger.console) + def test_search_tree(self): + # file match & ext miss + with temptree(['asdf.conf', 'blarg.conf', 'asdf.cfg']) as t: + asdf = utils.search_tree(t, 'a*', '.conf') + self.assertEquals(len(asdf), 1) + self.assertEquals(asdf[0], + os.path.join(t, 'asdf.conf')) + + # multi-file match & glob miss & sort + with temptree(['application.bin', 'apple.bin', 'apropos.bin']) as t: + app_bins = utils.search_tree(t, 'app*', 'bin') + self.assertEquals(len(app_bins), 2) + self.assertEquals(app_bins[0], + os.path.join(t, 'apple.bin')) + self.assertEquals(app_bins[1], + os.path.join(t, 'application.bin')) + + # test file in folder & ext miss & glob miss + files = ( + 'sub/file1.ini', + 'sub/file2.conf', + 'sub.bin', + 'bus.ini', + 'bus/file3.ini', + ) + with temptree(files) as t: + sub_ini = utils.search_tree(t, 'sub*', '.ini') + self.assertEquals(len(sub_ini), 1) + self.assertEquals(sub_ini[0], + os.path.join(t, 'sub/file1.ini')) + + # test multi-file in folder & sub-folder & ext miss & glob miss + files = ( + 'folder_file.txt', + 'folder/1.txt', + 'folder/sub/2.txt', + 'folder2/3.txt', + 'Folder3/4.txt' + 'folder.rc', + ) + with temptree(files) as t: + folder_texts = utils.search_tree(t, 'folder*', '.txt') + self.assertEquals(len(folder_texts), 4) + f1 = os.path.join(t, 'folder_file.txt') + f2 = os.path.join(t, 'folder/1.txt') + f3 = os.path.join(t, 'folder/sub/2.txt') + f4 = os.path.join(t, 'folder2/3.txt') + for f in [f1, f2, f3, f4]: + self.assert_(f in folder_texts) + + def test_write_file(self): + with temptree([]) as t: + file_name = os.path.join(t, 'test') + utils.write_file(file_name, 'test') + with open(file_name, 'r') as f: + contents = f.read() + self.assertEquals(contents, 'test') + # and also subdirs + file_name = os.path.join(t, 'subdir/test2') + utils.write_file(file_name, 'test2') + with open(file_name, 'r') as f: + contents = f.read() + self.assertEquals(contents, 'test2') + # but can't over-write files + file_name = os.path.join(t, 'subdir/test2/test3') + self.assertRaises(IOError, utils.write_file, file_name, + 'test3') + + def test_remove_file(self): + with temptree([]) as t: + file_name = os.path.join(t, 'blah.pid') + # assert no raise + self.assertEquals(os.path.exists(file_name), False) + self.assertEquals(utils.remove_file(file_name), None) + with open(file_name, 'w') as f: + f.write('1') + self.assert_(os.path.exists(file_name)) + self.assertEquals(utils.remove_file(file_name), None) + self.assertFalse(os.path.exists(file_name)) + + if __name__ == '__main__': unittest.main() From e3e604ec17cedf9a89242c5adc85edae9137f648 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Fri, 11 Feb 2011 13:21:28 -0600 Subject: [PATCH 44/64] forgot some new files --- swift/common/manager.py | 604 +++++++++++++ test/unit/common/test_manager.py | 1371 ++++++++++++++++++++++++++++++ 2 files changed, 1975 insertions(+) create mode 100644 swift/common/manager.py create mode 100644 test/unit/common/test_manager.py diff --git a/swift/common/manager.py b/swift/common/manager.py new file mode 100644 index 0000000000..089895044e --- /dev/null +++ b/swift/common/manager.py @@ -0,0 +1,604 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +import functools +import errno +import os +import resource +import signal +import sys +import time +from swift.common.utils import search_tree, remove_file, write_file +import subprocess +import re + +SWIFT_DIR = '/etc/swift' +RUN_DIR = '/var/run/swift' + +ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor', + 'container-replicator', 'container-server', 'container-updater', + 'object-auditor', 'object-server', 'object-replicator', 'object-updater', + 'proxy-server', 'account-replicator', 'auth-server', 'account-reaper'] +MAIN_SERVERS = ['auth-server', 'proxy-server', 'account-server', + 'container-server', 'object-server'] +REST_SERVERS = [s for s in ALL_SERVERS if s not in MAIN_SERVERS] +GRACEFUL_SHUTDOWN_SERVERS = MAIN_SERVERS +START_ONCE_SERVERS = REST_SERVERS + +KILL_WAIT = 15 # seconds to wait for servers to die + +MAX_DESCRIPTORS = 32768 +MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB + +def setup_env(): + """Try to increase resource limits of the OS. Move PYTHON_EGG_CACHE to /tmp + """ + try: + resource.setrlimit(resource.RLIMIT_NOFILE, + (MAX_DESCRIPTORS, MAX_DESCRIPTORS)) + resource.setrlimit(resource.RLIMIT_DATA, + (MAX_MEMORY, MAX_MEMORY)) + except ValueError: + print "WARNING: Unable to increase file descriptor limit." \ + " Running as non-root?" + + os.environ['PYTHON_EGG_CACHE'] = '/tmp' + return + +def command(func): + """ + Decorator to declare which methods are accessible as commands, commands + always return 1 or 0, where 0 should indicate success. + + :param func: function to make public + """ + func.publicly_accessible = True + + @functools.wraps(func) + def wrapped(*a, **kw): + rv = func(*a, **kw) + return 1 if rv else 0 + return wrapped + + +class UnknownCommandError(Exception): + pass + + +class Manager(): + """Main class for performing commands on groups of servers. + + :param servers: list of server names as strings + + """ + + def __init__(self, servers): + server_names = set() + for server in servers: + if server == 'all': + server_names.update(ALL_SERVERS) + elif server == 'main': + server_names.update(MAIN_SERVERS) + elif server == 'rest': + server_names.update(REST_SERVERS) + elif '*' in server: + # convert glob to regex + server_names.update([s for s in ALL_SERVERS if + re.match(server.replace('*', '.*'), s)]) + else: + server_names.add(server) + + self.servers = set() + for name in server_names: + self.servers.add(Server(name)) + + def watch_server_pids(self, server_pids, interval=0, **kwargs): + """Monitor a collection of server pids yeilding back those pids that + aren't responding to signals. + + :param server_pids: a dict, lists of pids [int,...] keyed on + Server objects + """ + status = {} + start = time.time() + end = start + interval + while interval: + for server, pids in server_pids.items(): + for pid in pids: + try: + # let pid stop if it wants to + os.waitpid(pid, os.WNOHANG) + except OSError, e: + if e.errno not in (errno.ECHILD, errno.ESRCH): + raise # else no such child/process + # check running pids for server + status[server] = server.get_running_pids(**kwargs) + for pid in pids: + # original pids no longer in running pids! + if pid not in status[server]: + yield server, pid + # update active pids list using running_pids + server_pids[server] = status[server] + if not [p for server, pids in status.items() for p in pids]: + # no more running pids + break + if time.time() > end: + break + else: + time.sleep(0.1) + return + + @command + def status(self, **kwargs): + """display status of tracked pids for server + """ + status = 0 + for server in self.servers: + status += server.status(**kwargs) + return status + + @command + def start(self, **kwargs): + """starts a server + """ + setup_env() + status = 0 + + for server in self.servers: + server.launch(**kwargs) + if not kwargs.get('daemon', True): + for server in self.servers: + try: + status += server.interact(**kwargs) + except KeyboardInterrupt: + print '\nuser quit' + self.stop(**kwargs) + break + elif kwargs.get('wait', False): + for server in self.servers: + status += server.wait(**kwargs) + return status + + @command + def wait(self, **kwargs): + """spawn server and wait for it to start + """ + kwargs['wait'] = True + return self.start(**kwargs) + + @command + def no_daemon(self, **kwargs): + """start a server interactivly + """ + kwargs['daemon'] = False + return self.start(**kwargs) + + @command + def once(self, **kwargs): + """start server and run one pass on supporting daemons + """ + kwargs['once'] = True + return self.start(**kwargs) + + @command + def stop(self, **kwargs): + """stops a server + """ + server_pids = {} + for server in self.servers: + signaled_pids = server.stop(**kwargs) + if not signaled_pids: + print 'No %s running' % server + else: + server_pids[server] = signaled_pids + + # all signaled_pids, i.e. list(itertools.chain(*server_pids.values())) + signaled_pids = [p for server, pid in server_pids.items() for p in pid] + # keep track of the pids yeiled back as killed for all servers + killed_pids = set() + for server, killed_pid in self.watch_server_pids(server_pids, + interval=KILL_WAIT, **kwargs): + print "%s (%s) appears to have stopped" % (server, killed_pid) + killed_pids.add(killed_pid) + if not killed_pids.symmetric_difference(signaled_pids): + # all proccesses have been stopped + return 0 + + # reached interval n watch_pids w/o killing all servers + for server, pids in server_pids.items(): + if not killed_pids.issuperset(pids): + # some pids of this server were not killed + print 'Waited 15 seconds for %s to die; giving up' % (server) + return 1 + + @command + def shutdown(self, **kwargs): + """allow current requests to finish on supporting servers + """ + kwargs['graceful'] = True + status = 0 + self.stop(**kwargs) + return status + + @command + def restart(self, **kwargs): + """stops then restarts server + """ + status = 0 + self.stop(**kwargs) + self.start(**kwargs) + return status + + @command + def reload(self, **kwargs): + """graceful shutdown then restart on supporting servers + """ + kwargs['graceful'] = True + status = 0 + for server in self.servers: + init = Manager([server.server]) + status += init.stop(**kwargs) + status += init.start(**kwargs) + return status + + @command + def force_reload(self, **kwargs): + """alias for reload + """ + return self.reload(**kwargs) + + def get_command(self, cmd): + """Find and return the decorated method named like cmd + + :param cmd: the command to get, a string, if not found raises + UnknownCommandError + + """ + cmd = cmd.lower().replace('-', '_') + try: + f = getattr(self, cmd) + except AttributeError: + raise UnknownCommandError(cmd) + if not hasattr(f, 'publicly_accessible'): + raise UnknownCommandError(cmd) + return f + + @classmethod + def list_commands(cls): + """Get all publicly accessible commands + + :returns: a list of string tuples (cmd, help), the method names who are + decorated as commands + """ + get_method = lambda cmd: getattr(cls, cmd) + return sorted([(x.replace('_', '-'), get_method(x).__doc__.strip()) + for x in dir(cls) if + getattr(get_method(x), 'publicly_accessible', False)]) + + def run_command(self, cmd, **kwargs): + """Find the named command and run it + + :param cmd: the command name to run + + """ + f = self.get_command(cmd) + return f(**kwargs) + + +class Server(): + """Manage operations on a server or group of servers of similar type + + :param server: name of server + """ + + def __init__(self, server): + if '-' not in server: + server = '%s-server' % server + self.server = server.lower() + self.type = '-'.join(server.split('-')[:-1]) + self.cmd = 'swift-%s' % server + self.procs = [] + + def __str__(self): + return self.server + + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, repr(str(self))) + + def __hash__(self): + return hash(str(self)) + + def __eq__(self, other): + try: + return self.server == other.server + except AttributeError: + return False + + def get_pid_file_name(self, ini_file): + """Translate ini_file to a corresponding pid_file + + :param ini_file: an ini_file for this server, a string + + :returns: the pid_file for this ini_file + + """ + return ini_file.replace( + os.path.normpath(SWIFT_DIR), RUN_DIR, 1).replace( + '%s-server' % self.type, self.server, 1).rsplit( + '.conf', 1)[0] + '.pid' + + def get_ini_file_name(self, pid_file): + """Translate pid_file to a corresponding ini_file + + :param pid_file: a pid_file for this server, a string + + :returns: the ini_file for this pid_file + + """ + return pid_file.replace( + os.path.normpath(RUN_DIR), SWIFT_DIR, 1).replace( + self.server, '%s-server' % self.type, 1).rsplit( + '.pid', 1)[0] + '.conf' + + def ini_files(self, **kwargs): + """Get ini files for this server + + :param: number, if supplied will only lookup the nth server + + :returns: list of ini files + """ + found_ini_files = search_tree(SWIFT_DIR, '%s-server*' % self.type, + '.conf') + number = kwargs.get('number') + if number: + try: + ini_files = [found_ini_files[number - 1]] + except IndexError: + ini_files = [] + else: + ini_files = found_ini_files + if not ini_files: + # maybe there's a config file(s) out there, but I couldn't find it! + if not kwargs.get('quiet'): + print('Unable to locate config %sfor %s' % ( + ('number %s ' % number if number else ''), self.server)) + if kwargs.get('verbose') and not kwargs.get('quiet'): + if found_ini_files: + print('Found configs:') + for i, ini_file in enumerate(found_ini_files): + print(' %d) %s' % (i + 1, ini_file)) + + return ini_files + + def pid_files(self, **kwargs): + """Get pid files for this server + + :param: number, if supplied will only lookup the nth server + + :returns: list of pid files + """ + pid_files = search_tree(RUN_DIR, '%s*' % self.server, '.pid') + number = kwargs.get('number', 0) + if number: + ini_files = self.ini_files(**kwargs) + # limt pid_files the one who translates to the indexed ini_file for + # this given number + pid_files = [pid_file for pid_file in pid_files if + self.get_ini_file_name(pid_file) in ini_files] + return pid_files + + def iter_pid_files(self, **kwargs): + """Generator, yields (pid_file, pids) + """ + for pid_file in self.pid_files(**kwargs): + yield pid_file, int(open(pid_file).read().strip()) + + def signal_pids(self, sig, **kwargs): + """Send a signal to pids for this server + + :param sig: signal to send + + :returns: a dict mapping pids (ints) to pid_files (paths) + + """ + pids = {} + for pid_file, pid in self.iter_pid_files(**kwargs): + try: + if sig != signal.SIG_DFL: + print 'Signal %s pid: %s signal: %s' % ( + self.server, pid, sig) + os.kill(pid, sig) + except OSError, e: + #print '%s sig err: %s' % (pid, e) + if e.errno == 3: + # pid does not exist + if kwargs.get('verbose'): + print "Removing stale pid file %s" % pid_file + remove_file(pid_file) + else: + # process exists + pids[pid] = pid_file + return pids + + def get_running_pids(self, **kwargs): + """Get running pids + + :returns: a dict mapping pids (ints) to pid_files (paths) + + """ + return self.signal_pids(signal.SIG_DFL, **kwargs) # send noop + + def kill_running_pids(self, **kwargs): + """Kill running pids + + :param graceful: if True, attempt SIGHUP on supporting servers + + :returns: a dict mapping pids (ints) to pid_files (paths) + + """ + graceful = kwargs.get('graceful') + if graceful and self.server in GRACEFUL_SHUTDOWN_SERVERS: + sig = signal.SIGHUP + else: + sig = signal.SIGTERM + return self.signal_pids(sig, **kwargs) + + def status(self, pids=None, **kwargs): + """Display status of server + + :param: pids, if not supplied pids will be populated automatically + :param: number, if supplied will only lookup the nth server + + :returns: 1 if server is not running, 0 otherwise + """ + if pids is None: + pids = self.get_running_pids(**kwargs) + if not pids: + number = kwargs.get('number', 0) + if number: + kwargs['quiet'] = True + ini_files = self.ini_files(**kwargs) + if ini_files: + print "%s #%d not running (%s)" % (self.server, number, + ini_files[0]) + else: + print "No %s running" % self.server + return 1 + for pid, pid_file in pids.items(): + ini_file = self.get_ini_file_name(pid_file) + print "%s running (%s - %s)" % (self.server, pid, ini_file) + return 0 + + def spawn(self, ini_file, once=False, wait=False, daemon=True, **kwargs): + """Launch a subprocess for this server. + + :param ini_file: path to ini_file to use as first arg + :param once: boolean, add once argument to command + :param wait: boolean, if true capture stdout with a pipe + :param daemon: boolean, if true ask server to log to console + + :returns : the pid of the spawned process + """ + args = [self.cmd, ini_file] + if once: + args.append('once') + if not daemon: + # ask the server to log to console + args.append('verbose') + + # figure out what we're going to do with stdio + if not daemon: + # do nothing, this process is open until the spawns close anyway + re_out = None + re_err = None + else: + re_err = subprocess.STDOUT + if wait: + # we're going to need to block on this... + re_out = subprocess.PIPE + else: + re_out = open(os.devnull, 'w+b') + proc = subprocess.Popen(args, stdout=re_out, stderr=re_err) + pid_file = self.get_pid_file_name(ini_file) + write_file(pid_file, proc.pid) + self.procs.append(proc) + return proc.pid + + def wait(self, **kwargs): + """ + wait on spawned procs to start + """ + status = 0 + for proc in self.procs: + # wait for process to close it's stdout + output = proc.stdout.read() + if output: + print output + if proc.returncode: + status += 1 + return status + + def interact(self, **kwargs): + """ + wait on spawned procs to terminate + """ + status = 0 + for proc in self.procs: + # wait for process to terminate + proc.communicate()[0] + if proc.returncode: + status += 1 + return status + + def launch(self, **kwargs): + """ + Collect ini files and attempt to spawn the processes for this server + """ + ini_files = self.ini_files(**kwargs) + if not ini_files: + return [] + + pids = self.get_running_pids(**kwargs) + + already_started = False + for pid, pid_file in pids.items(): + ini_file = self.get_ini_file_name(pid_file) + # for legacy compat you can't start other servers if one server is + # already running (unless -n specifies which one you want), this + # restriction could potentially be lifted, and launch could start + # any unstarted instances + if ini_file in ini_files: + already_started = True + print "%s running (%s - %s)" % (self.server, pid, ini_file) + elif not kwargs.get('number', 0): + already_started = True + print "%s running (%s - %s)" % (self.server, pid, pid_file) + + if already_started: + print "%s already started..." % self.server + #self.status(pids) + return [] + + if self.server not in START_ONCE_SERVERS: + kwargs['once'] = False + + # TODO: check if self.cmd exists? + + pids = {} + for ini_file in ini_files: + if kwargs.get('once'): + msg = 'Running %s once' % self.server + else: + msg = 'Starting %s' % self.server + print '%s...(%s)' % (msg, ini_file) + try: + pid = self.spawn(ini_file, **kwargs) + except OSError, e: + if e.errno == errno.ENOENT: + # cmd does not exist + print "%s does not exist" % self.cmd + break + pids[pid] = ini_file + + return pids + + def stop(self, **kwargs): + """Send stop signals to pids for this server + + :returns: a dict mapping pids (ints) to pid_files (paths) + + """ + return self.kill_running_pids(**kwargs) diff --git a/test/unit/common/test_manager.py b/test/unit/common/test_manager.py new file mode 100644 index 0000000000..a929b1731d --- /dev/null +++ b/test/unit/common/test_manager.py @@ -0,0 +1,1371 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from nose import SkipTest +from test.unit import temptree + +import os +import sys +import signal +import errno +from contextlib import contextmanager +from collections import defaultdict +from threading import Thread +from time import sleep, time + +from swift.common import manager + +DUMMY_SIG = 1 + +class MockOs(): + + def __init__(self, pids): + self.running_pids = pids + self.pid_sigs = defaultdict(list) + self.closed_fds = [] + self.child_pid = 9999 # fork defaults to test parent process path + self.execlp_called = False + + def kill(self, pid, sig): + if pid not in self.running_pids: + raise OSError(3, 'No such process') + self.pid_sigs[pid].append(sig) + + def __getattr__(self, name): + # I only over-ride portions of the os module + try: + return object.__getattr__(self, name) + except AttributeError: + return getattr(os, name) + + +def pop_stream(f): + """read everything out of file from the top and clear it out + """ + f.flush() + f.seek(0) + output = f.read() + f.seek(0) + f.truncate() + #print >> sys.stderr, output + return output + + +class TestMangerModule(unittest.TestCase): + + def test_servers(self): + main_plus_rest = set(manager.MAIN_SERVERS + manager.REST_SERVERS) + self.assertEquals(set(manager.ALL_SERVERS), main_plus_rest) + # make sure there's no server listed in both + self.assertEquals(len(main_plus_rest), len(manager.MAIN_SERVERS) + + len(manager.REST_SERVERS)) + + def test_setup_env(self): + # TODO: tests + raise SkipTest + + def test_command_wrapper(self): + @manager.command + def myfunc(arg1): + """test doc + """ + return arg1 + + self.assertEquals(myfunc.__doc__.strip(), 'test doc') + self.assertEquals(myfunc(1), 1) + self.assertEquals(myfunc(0), 0) + self.assertEquals(myfunc(True), 1) + self.assertEquals(myfunc(False), 0) + self.assert_(hasattr(myfunc, 'publicly_accessible')) + self.assert_(myfunc.publicly_accessible) + + def test_exc(self): + self.assert_(issubclass(manager.UnknownCommandError, Exception)) + +class TestServer(unittest.TestCase): + + def tearDown(self): + reload(manager) + + def join_swift_dir(self, path): + return os.path.join(manager.SWIFT_DIR, path) + + def join_run_dir(self, path): + return os.path.join(manager.RUN_DIR, path) + + def test_create_server(self): + server = manager.Server('proxy') + self.assertEquals(server.server, 'proxy-server') + self.assertEquals(server.type, 'proxy') + self.assertEquals(server.cmd, 'swift-proxy-server') + server = manager.Server('object-replicator') + self.assertEquals(server.server, 'object-replicator') + self.assertEquals(server.type, 'object') + self.assertEquals(server.cmd, 'swift-object-replicator') + + def test_server_to_string(self): + server = manager.Server('Proxy') + self.assertEquals(str(server), 'proxy-server') + server = manager.Server('object-replicator') + self.assertEquals(str(server), 'object-replicator') + + def test_server_repr(self): + server = manager.Server('proxy') + self.assert_(server.__class__.__name__ in repr(server)) + self.assert_(str(server) in repr(server)) + + def test_server_equality(self): + server1 = manager.Server('Proxy') + server2 = manager.Server('proxy-server') + self.assertEquals(server1, server2) + # it is NOT a string + self.assertNotEquals(server1, 'proxy-server') + + def test_get_pid_file_name(self): + server = manager.Server('proxy') + ini_file = self.join_swift_dir('proxy-server.conf') + pid_file = self.join_run_dir('proxy-server.pid') + self.assertEquals(pid_file, server.get_pid_file_name(ini_file)) + server = manager.Server('object-replicator') + ini_file = self.join_swift_dir('object-server/1.conf') + pid_file = self.join_run_dir('object-replicator/1.pid') + self.assertEquals(pid_file, server.get_pid_file_name(ini_file)) + server = manager.Server('container-auditor') + ini_file = self.join_swift_dir( + 'container-server/1/container-auditor.conf') + pid_file = self.join_run_dir( + 'container-auditor/1/container-auditor.pid') + self.assertEquals(pid_file, server.get_pid_file_name(ini_file)) + + def test_get_ini_file_name(self): + server = manager.Server('proxy') + ini_file = self.join_swift_dir('proxy-server.conf') + pid_file = self.join_run_dir('proxy-server.pid') + self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) + server = manager.Server('object-replicator') + ini_file = self.join_swift_dir('object-server/1.conf') + pid_file = self.join_run_dir('object-replicator/1.pid') + self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) + server = manager.Server('container-auditor') + ini_file = self.join_swift_dir( + 'container-server/1/container-auditor.conf') + pid_file = self.join_run_dir( + 'container-auditor/1/container-auditor.pid') + self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) + + def test_ini_files(self): + # test get single ini file + ini_files = ( + 'proxy-server.conf', + 'proxy-server.ini', + 'auth-server.conf', + ) + with temptree(ini_files) as t: + manager.SWIFT_DIR = t + server = manager.Server('proxy') + ini_files = server.ini_files() + self.assertEquals(len(ini_files), 1) + ini_file = ini_files[0] + proxy_conf = self.join_swift_dir('proxy-server.conf') + self.assertEquals(ini_file, proxy_conf) + + # test multi server conf files & grouping of server-type config + ini_files = ( + 'object-server1.conf', + 'object-server/2.conf', + 'object-server/object3.conf', + 'object-server/conf/server4.conf', + 'object-server.txt', + 'proxy-server.conf', + ) + with temptree(ini_files) as t: + manager.SWIFT_DIR = t + server = manager.Server('object-replicator') + ini_files = server.ini_files() + self.assertEquals(len(ini_files), 4) + c1 = self.join_swift_dir('object-server1.conf') + c2 = self.join_swift_dir('object-server/2.conf') + c3 = self.join_swift_dir('object-server/object3.conf') + c4 = self.join_swift_dir('object-server/conf/server4.conf') + for c in [c1, c2, c3, c4]: + self.assert_(c in ini_files) + # test configs returned sorted + sorted_confs = sorted([c1, c2, c3, c4]) + self.assertEquals(ini_files, sorted_confs) + + # test get single numbered conf + ini_files = ( + 'account-server/1.conf', + 'account-server/2.conf', + 'account-server/3.conf', + 'account-server/4.conf', + ) + with temptree(ini_files) as t: + manager.SWIFT_DIR = t + server = manager.Server('account') + ini_files = server.ini_files(number=2) + self.assertEquals(len(ini_files), 1) + ini_file = ini_files[0] + self.assertEquals(ini_file, + self.join_swift_dir('account-server/2.conf')) + # test missing config number + ini_files = server.ini_files(number=5) + self.assertFalse(ini_files) + + # test verbose & quiet + ini_files = ( + 'auth-server.ini', + 'container-server/1.conf', + ) + with temptree(ini_files) as t: + manager.SWIFT_DIR = t + old_stdout = sys.stdout + try: + with open(os.path.join(t, 'output'), 'w+') as f: + sys.stdout = f + server = manager.Server('auth') + # check warn "unable to locate" + ini_files = server.ini_files() + self.assertFalse(ini_files) + self.assert_('unable to locate' in pop_stream(f).lower()) + # check quiet will silence warning + ini_files = server.ini_files(verbose=True, quiet=True) + self.assertEquals(pop_stream(f), '') + # check found config no warning + server = manager.Server('container-auditor') + ini_files = server.ini_files() + self.assertEquals(pop_stream(f), '') + # check missing config number warn "unable to locate" + ini_files = server.ini_files(number=2) + self.assert_('unable to locate' in pop_stream(f).lower()) + # check verbose lists configs + ini_files = server.ini_files(number=2, verbose=True) + c1 = self.join_swift_dir('container-server/1.conf') + self.assert_(c1 in pop_stream(f)) + finally: + sys.stdout = old_stdout + + def test_iter_pid_files(self): + """ + Server.iter_pid_files is kinda boring, test the + Server.pid_files stuff here as well + """ + pid_files = ( + ('proxy-server.pid', 1), + ('auth-server.pid', 'blah'), + ('object-replicator/1.pid', 11), + ('object-replicator/2.pid', 12), + ) + files, contents = zip(*pid_files) + with temptree(files, contents) as t: + manager.RUN_DIR = t + server = manager.Server('proxy') + # test get one file + iter = server.iter_pid_files() + pid_file, pid = iter.next() + self.assertEquals(pid_file, self.join_run_dir('proxy-server.pid')) + self.assertEquals(pid, 1) + # ... and only one file + self.assertRaises(StopIteration, iter.next) + # test invalid value in pid file + server = manager.Server('auth') + self.assertRaises(ValueError, server.iter_pid_files().next) + # test object-server doesn't steal pids from object-replicator + server = manager.Server('object') + self.assertRaises(StopIteration, server.iter_pid_files().next) + # test multi-pid iter + server = manager.Server('object-replicator') + real_map = { + 11: self.join_run_dir('object-replicator/1.pid'), + 12: self.join_run_dir('object-replicator/2.pid'), + } + pid_map = {} + for pid_file, pid in server.iter_pid_files(): + pid_map[pid] = pid_file + self.assertEquals(pid_map, real_map) + + # test get pid_files by number + ini_files = ( + 'object-server/1.conf', + 'object-server/2.conf', + 'object-server/3.conf', + 'object-server/4.conf', + ) + + pid_files = ( + ('object-server/1.pid', 1), + ('object-server/2.pid', 2), + ('object-server/5.pid', 5), + ) + + with temptree(ini_files) as swift_dir: + manager.SWIFT_DIR = swift_dir + files, pids = zip(*pid_files) + with temptree(files, pids) as t: + manager.RUN_DIR = t + server = manager.Server('object') + # test get all pid files + real_map = { + 1: self.join_run_dir('object-server/1.pid'), + 2: self.join_run_dir('object-server/2.pid'), + 5: self.join_run_dir('object-server/5.pid'), + } + pid_map = {} + for pid_file, pid in server.iter_pid_files(): + pid_map[pid] = pid_file + self.assertEquals(pid_map, real_map) + # test get pid with matching conf + pids = list(server.iter_pid_files(number=2)) + self.assertEquals(len(pids), 1) + pid_file, pid = pids[0] + self.assertEquals(pid, 2) + pid_two = self.join_run_dir('object-server/2.pid') + self.assertEquals(pid_file, pid_two) + # try to iter on a pid number with a matching conf but no pid + pids = list(server.iter_pid_files(number=3)) + self.assertFalse(pids) + # test get pids w/o matching conf + pids = list(server.iter_pid_files(number=5)) + self.assertFalse(pids) + + def test_signal_pids(self): + pid_files = ( + ('proxy-server.pid', 1), + ('auth-server.pid', 2), + ) + files, pids = zip(*pid_files) + with temptree(files, pids) as t: + manager.RUN_DIR = t + # mock os with both pids running + manager.os = MockOs([1, 2]) + server = manager.Server('proxy') + pids = server.signal_pids(DUMMY_SIG) + self.assertEquals(len(pids), 1) + self.assert_(1 in pids) + self.assertEquals(manager.os.pid_sigs[1], [DUMMY_SIG]) + # make sure other process not signaled + self.assertFalse(2 in pids) + self.assertFalse(2 in manager.os.pid_sigs) + # capture stdio + old_stdout = sys.stdout + try: + with open(os.path.join(t, 'output'), 'w+') as f: + sys.stdout = f + #test print details + pids = server.signal_pids(DUMMY_SIG) + output = pop_stream(f) + self.assert_('pid: %s' % 1 in output) + self.assert_('signal: %s' % DUMMY_SIG in output) + # test no details on signal.SIG_DFL + pids = server.signal_pids(signal.SIG_DFL) + self.assertEquals(pop_stream(f), '') + # reset mock os so only the other server is running + manager.os = MockOs([2]) + # test pid not running + pids = server.signal_pids(signal.SIG_DFL) + self.assert_(1 not in pids) + self.assert_(1 not in manager.os.pid_sigs) + # test remove stale pid file + self.assertFalse(os.path.exists( + self.join_run_dir('proxy-server.pid'))) + # reset mock os with no running pids + manager.os = MockOs([]) + server = manager.Server('auth') + # test verbose warns on removing pid file + pids = server.signal_pids(signal.SIG_DFL, verbose=True) + output = pop_stream(f) + self.assert_('stale pid' in output.lower()) + auth_pid = self.join_run_dir('auth-server.pid') + self.assert_(auth_pid in output) + finally: + sys.stdout = old_stdout + + def test_get_running_pids(self): + # test only gets running pids + pid_files = ( + ('test-server1.pid', 1), + ('test-server2.pid', 2), + ) + with temptree(*zip(*pid_files)) as t: + manager.RUN_DIR = t + server = manager.Server('test-server') + # mock os, only pid '1' is running + manager.os = MockOs([1]) + running_pids = server.get_running_pids() + self.assertEquals(len(running_pids), 1) + self.assert_(1 in running_pids) + self.assert_(2 not in running_pids) + # test persistant running pid files + self.assert_(os.path.exists(os.path.join(t, 'test-server1.pid'))) + # test clean up stale pids + pid_two = self.join_swift_dir('test-server2.pid') + self.assertFalse(os.path.exists(pid_two)) + # reset mock os, no pids running + manager.os = MockOs([]) + running_pids = server.get_running_pids() + self.assertFalse(running_pids) + # and now all pid files are cleaned out + pid_one = self.join_run_dir('test-server1.pid') + self.assertFalse(os.path.exists(pid_one)) + all_pids = os.listdir(t) + self.assertEquals(len(all_pids), 0) + + # test only get pids for right server + pid_files = ( + ('thing-doer.pid', 1), + ('thing-sayer.pid', 2), + ('other-doer.pid', 3), + ('other-sayer.pid', 4), + ) + files, pids = zip(*pid_files) + with temptree(files, pids) as t: + manager.RUN_DIR = t + # all pids are running + manager.os = MockOs(pids) + server = manager.Server('thing-doer') + running_pids = server.get_running_pids() + # only thing-doer.pid, 1 + self.assertEquals(len(running_pids), 1) + self.assert_(1 in running_pids) + # no other pids returned + for n in (2, 3, 4): + self.assert_(n not in running_pids) + # assert stale pids for other servers ignored + manager.os = MockOs([1]) # only thing-doer is running + running_pids = server.get_running_pids() + for f in ('thing-sayer.pid', 'other-doer.pid', 'other-sayer.pid'): + # other server pid files persist + self.assert_(os.path.exists, os.path.join(t, f)) + # verify that servers are in fact not running + for server_name in ('thing-sayer', 'other-doer', 'other-sayer'): + server = manager.Server(server_name) + running_pids = server.get_running_pids() + self.assertFalse(running_pids) + # and now all OTHER pid files are cleaned out + all_pids = os.listdir(t) + self.assertEquals(len(all_pids), 1) + self.assert_(os.path.exists(os.path.join(t, 'thing-doer.pid'))) + + def test_kill_running_pids(self): + pid_files = ( + ('object-server.pid', 1), + ('object-replicator1.pid', 11), + ('object-replicator2.pid', 12), + ) + files, running_pids = zip(*pid_files) + with temptree(files, running_pids) as t: + manager.RUN_DIR = t + server = manager.Server('object') + # test no servers running + manager.os = MockOs([]) + pids = server.kill_running_pids() + self.assertFalse(pids, pids) + files, running_pids = zip(*pid_files) + with temptree(files, running_pids) as t: + manager.RUN_DIR = t + # start up pid + manager.os = MockOs([1]) + # test kill one pid + pids = server.kill_running_pids() + self.assertEquals(len(pids), 1) + self.assert_(1 in pids) + self.assertEquals(manager.os.pid_sigs[1], [signal.SIGTERM]) + # reset os mock + manager.os = MockOs([1]) + # test shutdown + self.assert_('object-server' in + manager.GRACEFUL_SHUTDOWN_SERVERS) + pids = server.kill_running_pids(graceful=True) + self.assertEquals(len(pids), 1) + self.assert_(1 in pids) + self.assertEquals(manager.os.pid_sigs[1], [signal.SIGHUP]) + # start up other servers + manager.os = MockOs([11, 12]) + # test multi server kill & ignore graceful on unsupport server + self.assertFalse('object-replicator' in + manager.GRACEFUL_SHUTDOWN_SERVERS) + server = manager.Server('object-replicator') + pids = server.kill_running_pids(graceful=True) + self.assertEquals(len(pids), 2) + for pid in (11, 12): + self.assert_(pid in pids) + self.assertEquals(manager.os.pid_sigs[pid], + [signal.SIGTERM]) + # and the other pid is of course not signaled + self.assert_(1 not in manager.os.pid_sigs) + + def test_status(self): + ini_files = ( + 'test-server/1.conf', + 'test-server/2.conf', + 'test-server/3.conf', + 'test-server/4.conf', + ) + + pid_files = ( + ('test-server/1.pid', 1), + ('test-server/2.pid', 2), + ('test-server/3.pid', 3), + ('test-server/4.pid', 4), + ) + + with temptree(ini_files) as swift_dir: + manager.SWIFT_DIR = swift_dir + files, pids = zip(*pid_files) + with temptree(files, pids) as t: + manager.RUN_DIR = t + # setup running servers + server = manager.Server('test') + # capture stdio + old_stdout = sys.stdout + try: + with open(os.path.join(t, 'output'), 'w+') as f: + sys.stdout = f + # test status for all running + manager.os = MockOs(pids) + self.assertEquals(server.status(), 0) + output = pop_stream(f).strip().splitlines() + self.assertEquals(len(output), 4) + for line in output: + self.assert_('test-server running' in line) + # test get single server by number + self.assertEquals(server.status(number=4), 0) + output = pop_stream(f).strip().splitlines() + self.assertEquals(len(output), 1) + line = output[0] + self.assert_('test-server running' in line) + conf_four = self.join_swift_dir(ini_files[3]) + self.assert_('4 - %s' % conf_four in line) + # test some servers not running + manager.os = MockOs([1, 2, 3]) + self.assertEquals(server.status(), 0) + output = pop_stream(f).strip().splitlines() + self.assertEquals(len(output), 3) + for line in output: + self.assert_('test-server running' in line) + # test single server not running + manager.os = MockOs([1, 2]) + self.assertEquals(server.status(number=3), 1) + output = pop_stream(f).strip().splitlines() + self.assertEquals(len(output), 1) + line = output[0] + self.assert_('not running' in line) + conf_three = self.join_swift_dir(ini_files[2]) + self.assert_(conf_three in line) + # test no running pids + manager.os = MockOs([]) + self.assertEquals(server.status(), 1) + output = pop_stream(f).lower() + self.assert_('no test-server running' in output) + # test use provided pids + pids = { + 1: '1.pid', + 2: '2.pid', + } + # shouldn't call get_running_pids + called = [] + + def mock(*args, **kwargs): + called.append(True) + server.get_running_pids = mock + status = server.status(pids=pids) + self.assertEquals(status, 0) + self.assertFalse(called) + output = pop_stream(f).strip().splitlines() + self.assertEquals(len(output), 2) + for line in output: + self.assert_('test-server running' in line) + finally: + sys.stdout = old_stdout + + def test_spawn(self): + + # mocks + class MockProcess(): + + NOTHING = 'default besides None' + STDOUT = 'stdout' + PIPE = 'pipe' + + def __init__(self, pids=None): + if pids is None: + pids = [] + self.pids = (p for p in pids) + + def Popen(self, args, **kwargs): + return MockProc(self.pids.next(), args, **kwargs) + + class MockProc(): + + def __init__(self, pid, args, stdout=MockProcess.NOTHING, + stderr=MockProcess.NOTHING): + self.pid = pid + self.args = args + self.stdout = stdout + if stderr == MockProcess.STDOUT: + self.stderr = self.stdout + else: + self.stderr = stderr + + # setup running servers + server = manager.Server('test') + + with temptree(['test-server.conf']) as swift_dir: + manager.SWIFT_DIR = swift_dir + with temptree([]) as t: + manager.RUN_DIR = t + old_subprocess = manager.subprocess + try: + # test single server process calls spawn once + manager.subprocess = MockProcess([1]) + conf_file = self.join_swift_dir('test-server.conf') + # spawn server no kwargs + server.spawn(conf_file) + # test pid file + pid_file = self.join_run_dir('test-server.pid') + self.assert_(os.path.exists(pid_file)) + pid_on_disk = int(open(pid_file).read().strip()) + self.assertEquals(pid_on_disk, 1) + # assert procs args + self.assert_(server.procs) + self.assertEquals(len(server.procs), 1) + proc = server.procs[0] + expected_args = [ + 'swift-test-server', + conf_file, + ] + self.assertEquals(proc.args, expected_args) + # assert stdout is /dev/null + self.assert_(isinstance(proc.stdout, file)) + self.assertEquals(proc.stdout.name, os.devnull) + self.assertEquals(proc.stdout.mode, 'w+b') + self.assertEquals(proc.stderr, proc.stdout) + # test multi server process calls spawn multiple times + manager.subprocess = MockProcess([11, 12, 13, 14]) + conf1 = self.join_swift_dir('test-server/1.conf') + conf2 = self.join_swift_dir('test-server/2.conf') + conf3 = self.join_swift_dir('test-server/3.conf') + conf4 = self.join_swift_dir('test-server/4.conf') + server = manager.Server('test') + # test server run once + server.spawn(conf1, once=True) + self.assert_(server.procs) + self.assertEquals(len(server.procs), 1) + proc = server.procs[0] + expected_args = ['swift-test-server', conf1, 'once'] + self.assertEquals(proc.args, expected_args) + # assert stdout is /dev/null + self.assert_(isinstance(proc.stdout, file)) + self.assertEquals(proc.stdout.name, os.devnull) + self.assertEquals(proc.stdout.mode, 'w+b') + self.assertEquals(proc.stderr, proc.stdout) + # test server not daemon + server.spawn(conf2, daemon=False) + self.assert_(server.procs) + self.assertEquals(len(server.procs), 2) + proc = server.procs[1] + expected_args = ['swift-test-server', conf2, 'verbose'] + self.assertEquals(proc.args, expected_args) + # assert stdout is not changed + self.assertEquals(proc.stdout, None) + self.assertEquals(proc.stderr, None) + # test server wait + server.spawn(conf3, wait=True) + self.assert_(server.procs) + self.assertEquals(len(server.procs), 3) + proc = server.procs[2] + # assert stdout is piped + self.assertEquals(proc.stdout, MockProcess.PIPE) + self.assertEquals(proc.stderr, proc.stdout) + # test not daemon over-rides wait + server.spawn(conf4, wait=True, daemon=False, once=True) + self.assert_(server.procs) + self.assertEquals(len(server.procs), 4) + proc = server.procs[3] + expected_args = ['swift-test-server', conf4, 'once', + 'verbose'] + self.assertEquals(proc.args, expected_args) + # daemon behavior should trump wait, once shouldn't matter + self.assertEquals(proc.stdout, None) + self.assertEquals(proc.stderr, None) + # assert pids + for i, proc in enumerate(server.procs): + pid_file = self.join_run_dir('test-server/%d.pid' % + (i + 1)) + pid_on_disk = int(open(pid_file).read().strip()) + self.assertEquals(pid_on_disk, proc.pid) + finally: + manager.subprocess = old_subprocess + + def test_wait(self): + server = manager.Server('test') + self.assertEquals(server.wait(), 0) + + class MockProcess(Thread): + def __init__(self, delay=0.1, fail_to_start=False): + Thread.__init__(self) + # setup pipe + rfd, wfd = os.pipe() + # subprocess connection to read stdout + self.stdout = os.fdopen(rfd) + # real process connection to write stdout + self._stdout = os.fdopen(wfd, 'w') + self.delay = delay + self.finished = False + self.returncode = None + if fail_to_start: + self.run = self.fail + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + if self.isAlive(): + self.join() + + def close_stdout(self): + self._stdout.flush() + with open(os.devnull, 'wb') as nullfile: + try: + os.dup2(nullfile.fileno(), self._stdout.fileno()) + except OSError: + pass + + def fail(self): + print >>self._stdout, 'mock process started' + sleep(self.delay) # perform setup processing + print >>self._stdout, 'mock process failed to start' + self.returncode = 1 + self.close_stdout() + + def run(self): + print >>self._stdout, 'mock process started' + sleep(self.delay) # perform setup processing + print >>self._stdout, 'setup complete!' + self.close_stdout() + sleep(self.delay) # do some more processing + print >>self._stdout, 'mock process finished' + self.finished = True + + with temptree([]) as t: + old_stdout = sys.stdout + try: + with open(os.path.join(t, 'output'), 'w+') as f: + # acctually capture the read stdout (for prints) + sys.stdout = f + # test closing pipe in subprocess unblocks read + with MockProcess() as proc: + server.procs = [proc] + status = server.wait() + self.assertEquals(status, 0) + # wait should return as soon as stdout is closed + self.assert_(proc.isAlive()) + self.assertFalse(proc.finished) + self.assert_(proc.finished) # make sure it did finish... + # test output kwarg prints subprocess output + with MockProcess() as proc: + server.procs = [proc] + status = server.wait(output=True) + output = pop_stream(f) + self.assert_('mock process started' in output) + self.assert_('setup complete' in output) + # make sure we don't get prints after stdout was closed + self.assert_('mock process finished' not in output) + # test process which fails to start + with MockProcess(fail_to_start=True) as proc: + server.procs = [proc] + status = server.wait() + self.assertEquals(status, 1) + self.assert_('failed' in pop_stream(f)) + # test multiple procs + procs = [MockProcess() for i in range(3)] + for proc in procs: + proc.start() + server.procs = procs + status = server.wait() + self.assertEquals(status, 0) + for proc in procs: + self.assert_(proc.isAlive()) + for proc in procs: + proc.join() + finally: + sys.stdout = old_stdout + + def test_interact(self): + class MockProcess(): + + def __init__(self, fail=False): + self.returncode = None + if fail: + self._returncode = 1 + else: + self._returncode = 0 + + def communicate(self): + self.returncode = self._returncode + return '', '' + + server = manager.Server('test') + server.procs = [MockProcess()] + self.assertEquals(server.interact(), 0) + server.procs = [MockProcess(fail=True)] + self.assertEquals(server.interact(), 1) + procs = [] + for fail in (False, True, True): + procs.append(MockProcess(fail=fail)) + server.procs = procs + self.assert_(server.interact() > 0) + + def test_launch(self): + # stubs + ini_files = ( + 'proxy-server.conf', + 'object-server/1.conf', + 'object-server/2.conf', + 'object-server/3.conf', + 'object-server/4.conf', + ) + pid_files = ( + ('proxy-server.pid', 1), + ('proxy-server/2.pid', 2), + ) + + #mocks + class MockSpawn(): + + def __init__(self, pids=None): + self.ini_files = [] + self.kwargs = [] + if not pids: + def one_forever(): + while True: + yield 1 + self.pids = one_forever() + else: + self.pids = (x for x in pids) + + def __call__(self, ini_file, **kwargs): + self.ini_files.append(ini_file) + self.kwargs.append(kwargs) + return self.pids.next() + + with temptree(ini_files) as swift_dir: + manager.SWIFT_DIR = swift_dir + files, pids = zip(*pid_files) + with temptree(files, pids) as t: + manager.RUN_DIR = t + old_stdout = sys.stdout + try: + with open(os.path.join(t, 'output'), 'w+') as f: + sys.stdout = f + # can't start server w/o an conf + server = manager.Server('test') + self.assertFalse(server.launch()) + # start mock os running all pids + manager.os = MockOs(pids) + server = manager.Server('proxy') + # can't start server if it's already running + self.assertFalse(server.launch()) + output = pop_stream(f) + self.assert_('running' in output) + ini_file = self.join_swift_dir('proxy-server.conf') + self.assert_(ini_file in output) + pid_file = self.join_run_dir('proxy-server/2.pid') + self.assert_(pid_file in output) + self.assert_('already started' in output) + # no running pids + manager.os = MockOs([]) + # test ignore once for non-start-once server + mock_spawn = MockSpawn([1]) + server.spawn = mock_spawn + ini_file = self.join_swift_dir('proxy-server.conf') + expected = { + 1: ini_file, + } + self.assertEquals(server.launch(once=True), expected) + self.assertEquals(mock_spawn.ini_files, [ini_file]) + expected = { + 'once': False, + } + self.assertEquals(mock_spawn.kwargs, [expected]) + output = pop_stream(f) + self.assert_('Starting' in output) + self.assert_('once' not in output) + # test multi-server kwarg once + server = manager.Server('object-replicator') + mock_spawn = MockSpawn([1, 2, 3, 4]) + server.spawn = mock_spawn + conf1 = self.join_swift_dir('object-server/1.conf') + conf2 = self.join_swift_dir('object-server/2.conf') + conf3 = self.join_swift_dir('object-server/3.conf') + conf4 = self.join_swift_dir('object-server/4.conf') + expected = { + 1: conf1, + 2: conf2, + 3: conf3, + 4: conf4, + } + self.assertEquals(server.launch(once=True), expected) + self.assertEquals(mock_spawn.ini_files, [conf1, conf2, + conf3, conf4]) + expected = { + 'once': True, + } + self.assertEquals(len(mock_spawn.kwargs), 4) + for kwargs in mock_spawn.kwargs: + self.assertEquals(kwargs, expected) + # test number kwarg + mock_spawn = MockSpawn([4]) + server.spawn = mock_spawn + expected = { + 4: conf4, + } + self.assertEquals(server.launch(number=4), expected) + self.assertEquals(mock_spawn.ini_files, [conf4]) + expected = { + 'number': 4 + } + self.assertEquals(mock_spawn.kwargs, [expected]) + + + finally: + sys.stdout = old_stdout + + def test_stop(self): + ini_files = ( + 'account-server/1.conf', + 'account-server/2.conf', + 'account-server/3.conf', + 'account-server/4.conf', + ) + pid_files = ( + ('account-reaper/1.pid', 1), + ('account-reaper/2.pid', 2), + ('account-reaper/3.pid', 3), + ('account-reaper/4.pid', 4), + ) + + with temptree(ini_files) as swift_dir: + manager.SWIFT_DIR = swift_dir + files, pids = zip(*pid_files) + with temptree(files, pids) as t: + manager.RUN_DIR = t + # start all pids in mock os + manager.os = MockOs(pids) + server = manager.Server('account-reaper') + # test kill all running pids + pids = server.stop() + self.assertEquals(len(pids), 4) + for pid in (1, 2, 3, 4): + self.assert_(pid in pids) + self.assertEquals(manager.os.pid_sigs[pid], + [signal.SIGTERM]) + conf1 = self.join_swift_dir('account-reaper/1.conf') + conf2 = self.join_swift_dir('account-reaper/2.conf') + conf3 = self.join_swift_dir('account-reaper/3.conf') + conf4 = self.join_swift_dir('account-reaper/4.conf') + # reset mock os with only 2 running pids + manager.os = MockOs([3, 4]) + pids = server.stop() + self.assertEquals(len(pids), 2) + for pid in (3, 4): + self.assert_(pid in pids) + self.assertEquals(manager.os.pid_sigs[pid], + [signal.SIGTERM]) + self.assertFalse(os.path.exists(conf1)) + self.assertFalse(os.path.exists(conf2)) + # test number kwarg + manager.os = MockOs([3, 4]) + pids = server.stop(number=3) + self.assertEquals(len(pids), 1) + expected = { + 3: conf3, + } + self.assert_(pids, expected) + self.assertEquals(manager.os.pid_sigs[3], [signal.SIGTERM]) + self.assertFalse(os.path.exists(conf4)) + self.assertFalse(os.path.exists(conf3)) + +class TestManager(unittest.TestCase): + + def test_create(self): + controller = manager.Manager(['test']) + self.assertEquals(len(controller.servers), 1) + server = controller.servers.pop() + self.assert_(isinstance(server, manager.Server)) + self.assertEquals(server.server, 'test-server') + # test multi-server and simple dedupe + servers = ['object-replicator', 'object-auditor', 'object-replicator'] + controller = manager.Manager(servers) + self.assertEquals(len(controller.servers), 2) + for server in controller.servers: + self.assert_(server.server in servers) + # test all + controller = manager.Manager(['all']) + self.assertEquals(len(controller.servers), len(manager.ALL_SERVERS)) + for server in controller.servers: + self.assert_(server.server in manager.ALL_SERVERS) + # test main + controller = manager.Manager(['main']) + self.assertEquals(len(controller.servers), len(manager.MAIN_SERVERS)) + for server in controller.servers: + self.assert_(server.server in manager.MAIN_SERVERS) + # test rest + controller = manager.Manager(['rest']) + self.assertEquals(len(controller.servers), len(manager.REST_SERVERS)) + for server in controller.servers: + self.assert_(server.server in manager.REST_SERVERS) + # test main + rest == all + controller = manager.Manager(['main', 'rest']) + self.assertEquals(len(controller.servers), len(manager.ALL_SERVERS)) + for server in controller.servers: + self.assert_(server.server in manager.ALL_SERVERS) + # test dedupe + controller = manager.Manager(['main', 'rest', 'proxy', 'object', + 'container', 'account']) + self.assertEquals(len(controller.servers), len(manager.ALL_SERVERS)) + for server in controller.servers: + self.assert_(server.server in manager.ALL_SERVERS) + + #TODO: more tests + def test_watch_server_pids(self): + class MockOs(): + WNOHANG = os.WNOHANG + def __init__(self, pid_map={}): + self.pid_map = {} + for pid,v in pid_map.items(): + self.pid_map[pid] = (x for x in v) + def waitpid(self, pid, options): + try: + rv = self.pid_map[pid].next() + except StopIteration: + raise OSError(errno.ECHILD, os.strerror(errno.ECHILD)) + except KeyError: + raise OSError(errno.ESRCH, os.strerror(errno.ESRCH)) + if isinstance(rv, Exception): + raise rv() + else: + return rv + + class MockTime(): + def __init__(self, ticks=None): + self.tock = time() + if ticks: + ticks = [t for t in ticks] + else: + ticks = [] + + self.ticks = (t for t in ticks) + + def time(self): + try: + self.tock += self.ticks.next() + except StopIteration: + self.tock += 1 + return self.tock + + class MockServer(): + def __init__(self, server): + self.server = server + def get_running_pids(): + pass + + _orig_os = manager.os + _orig_time = manager.time + _orig_server= manager.Server + try: + manager.os = MockOs() + manager.time = MockTime() + manager.Server = MockServer + controller = manager.Manager(['test']) + self.assertEquals(len(controller.servers), 1) + server = controller.servers.pop() + server_pids = { + server: [1], + } + # test default interval + gen = controller.watch_server_pids(server_pids) + self.assertEquals([x for x in gen], []) + # test small interval + gen = controller.watch_server_pids(server_pids, interval=1) + # self.assertEquals([x for x in gen], []) + finally: + manager.os = _orig_os + manager.time = _orig_time + manager.Server = _orig_server + + def test_get_command(self): + raise SkipTest + + def test_list_commands(self): + for cmd, help in manager.Manager.list_commands(): + method = getattr(manager.Manager, cmd.replace('-', '_'), None) + self.assert_(method, '%s is not a command' % cmd) + self.assert_(getattr(method, 'publicly_accessible', False)) + self.assertEquals(method.__doc__.strip(), help) + + def test_run_command(self): + raise SkipTest + + def test_status(self): + class MockServer(): + def __init__(self, server): + self.server = server + self.called_kwargs = [] + def status(self, **kwargs): + self.called_kwargs.append(kwargs) + if 'error' in self.server: + return 1 + else: + return 0 + + old_server_class = manager.Server + try: + manager.Server = MockServer + controller = manager.Manager(['test']) + status = controller.status() + self.assertEquals(status, 0) + controller = manager.Manager(['error']) + status = controller.status() + self.assertEquals(status, 1) + # test multi-server + controller = manager.Manager(['test', 'error']) + kwargs = {'key': 'value'} + status = controller.status(**kwargs) + self.assertEquals(status, 1) + for server in controller.servers: + self.assertEquals(server.called_kwargs, [kwargs]) + finally: + manager.Server = old_server_class + + def test_start(self): + def mock_setup_env(): + getattr(mock_setup_env, 'called', []).append(True) + class MockServer(): + + def __init__(self, server): + self.server = server + self.called = defaultdict(list) + + def launch(self, **kwargs): + self.called['launch'].append(kwargs) + + def wait(self, **kwargs): + self.called['wait'].append(kwargs) + if 'error' in self.server: + return 1 + else: + return 0 + + def interact(self, **kwargs): + self.called['interact'].append(kwargs) + # TODO: test user quit + """ + if 'raise' in self.server: + raise KeyboardInterrupt + el + """ + if 'error' in self.server: + return 1 + else: + return 0 + + old_setup_env = manager.setup_env + old_swift_server = manager.Server + try: + manager.setup_env = mock_setup_env + manager.Server = MockServer + + # test no errors on launch + controller = manager.Manager(['proxy', 'error']) + status = controller.start() + self.assertEquals(status, 0) + for server in controller.servers: + self.assertEquals(server.called['launch'], [{}]) + + # test error on wait + controller = manager.Manager(['proxy', 'error']) + kwargs = {'wait': True} + status = controller.start(**kwargs) + self.assertEquals(status, 1) + for server in controller.servers: + self.assertEquals(server.called['launch'], [kwargs]) + self.assertEquals(server.called['wait'], [kwargs]) + + # test interact + controller = manager.Manager(['proxy', 'error']) + kwargs = {'daemon': False} + status = controller.start(**kwargs) + self.assertEquals(status, 1) + for server in controller.servers: + self.assertEquals(server.called['launch'], [kwargs]) + self.assertEquals(server.called['interact'], [kwargs]) + finally: + manager.setup_env = old_setup_env + manager.Server = old_swift_server + + + def test_wait(self): + class MockServer(): + def __init__(self, server): + self.server = server + self.called = defaultdict(list) + + def launch(self, **kwargs): + self.called['launch'].append(kwargs) + + def wait(self, **kwargs): + self.called['wait'].append(kwargs) + return int('error' in self.server) + + orig_swift_server = manager.Server + try: + manager.Server = MockServer + # test success + init = manager.Manager(['proxy']) + status = init.wait() + self.assertEquals(status, 0) + for server in init.servers: + self.assertEquals(len(server.called['launch']), 1) + called_kwargs = server.called['launch'][0] + self.assert_('wait' in called_kwargs) + self.assert_(called_kwargs['wait']) + self.assertEquals(len(server.called['wait']), 1) + called_kwargs = server.called['wait'][0] + self.assert_('wait' in called_kwargs) + self.assert_(called_kwargs['wait']) + # test error + init = manager.Manager(['error']) + status = init.wait() + self.assertEquals(status, 1) + for server in init.servers: + self.assertEquals(len(server.called['launch']), 1) + called_kwargs = server.called['launch'][0] + self.assert_('wait' in called_kwargs) + self.assert_(called_kwargs['wait']) + self.assertEquals(len(server.called['wait']), 1) + called_kwargs = server.called['wait'][0] + self.assert_('wait' in called_kwargs) + self.assert_(called_kwargs['wait']) + # test wait with once option + init = manager.Manager(['updater', 'replicator-error']) + status = init.wait(once=True) + self.assertEquals(status, 1) + for server in init.servers: + self.assertEquals(len(server.called['launch']), 1) + called_kwargs = server.called['launch'][0] + self.assert_('wait' in called_kwargs) + self.assert_(called_kwargs['wait']) + self.assert_('once' in called_kwargs) + self.assert_(called_kwargs['once']) + self.assertEquals(len(server.called['wait']), 1) + called_kwargs = server.called['wait'][0] + self.assert_('wait' in called_kwargs) + self.assert_(called_kwargs['wait']) + self.assert_('once' in called_kwargs) + self.assert_(called_kwargs['once']) + finally: + manager.Server = orig_swift_server + + + def test_no_daemon(self): + class MockServer(): + + def __init__(self, server): + self.server = server + self.called = defaultdict(list) + + def launch(self, **kwargs): + self.called['launch'].append(kwargs) + + def interact(self, **kwargs): + self.called['interact'].append(kwargs) + return int('error' in self.server) + + orig_swift_server = manager.Server + try: + manager.Server = MockServer + # test success + init = manager.Manager(['proxy']) + stats = init.no_daemon() + self.assertEquals(stats, 0) + # test error + init = manager.Manager(['proxy', 'object-error']) + stats = init.no_daemon() + self.assertEquals(stats, 1) + # test once + init = manager.Manager(['proxy', 'object-error']) + stats = init.no_daemon() + for server in init.servers: + self.assertEquals(len(server.called['launch']), 1) + self.assertEquals(len(server.called['wait']), 0) + self.assertEquals(len(server.called['interact']), 1) + finally: + manager.Server = orig_swift_server + + def test_once(self): + class MockServer(): + + def __init__(self, server): + self.server = server + self.called = defaultdict(list) + + def launch(self, **kwargs): + return self.called['launch'].append(kwargs) + + + orig_swift_server = manager.Server + try: + manager.Server = MockServer + # test no errors + init = manager.Manager(['account-reaper']) + status = init.once() + self.assertEquals(status, 0) + # test no error code on error + init = manager.Manager(['error-reaper']) + status = init.once() + self.assertEquals(status, 0) + for server in init.servers: + self.assertEquals(len(server.called['launch']), 1) + called_kwargs = server.called['launch'][0] + self.assertEquals(called_kwargs, {'once': True}) + self.assertEquals(len(server.called['wait']), 0) + self.assertEquals(len(server.called['interact']), 0) + finally: + manager.Server = orig_swift_server + + + def test_stop(self): + raise SkipTest + + def test_shutdown(self): + raise SkipTest + + def test_restart(self): + raise SkipTest + + def test_reload(self): + raise SkipTest + + def test_force_reload(self): + raise SkipTest + + + +if __name__ == '__main__': + unittest.main() + From 4c809e49b9b4b01b28516b7fa6d8d79df039ccae Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Fri, 11 Feb 2011 14:56:03 -0600 Subject: [PATCH 45/64] working on tests --- .bintests | 2 - test/unit/common/test_manager.py | 159 +++++++++++++++++++++---------- 2 files changed, 110 insertions(+), 51 deletions(-) delete mode 100755 .bintests diff --git a/.bintests b/.bintests deleted file mode 100755 index 55a94b303a..0000000000 --- a/.bintests +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -nosetests test/bin -v diff --git a/test/unit/common/test_manager.py b/test/unit/common/test_manager.py index a929b1731d..c48829d2c7 100644 --- a/test/unit/common/test_manager.py +++ b/test/unit/common/test_manager.py @@ -19,6 +19,7 @@ from test.unit import temptree import os import sys +import resource import signal import errno from contextlib import contextmanager @@ -64,7 +65,7 @@ def pop_stream(f): return output -class TestMangerModule(unittest.TestCase): +class TestManagerModule(unittest.TestCase): def test_servers(self): main_plus_rest = set(manager.MAIN_SERVERS + manager.REST_SERVERS) @@ -74,8 +75,54 @@ class TestMangerModule(unittest.TestCase): len(manager.REST_SERVERS)) def test_setup_env(self): - # TODO: tests - raise SkipTest + class MockResource(): + def __init__(self, error=None): + self.error = error + self.called_with_args = [] + + def setrlimit(self, resource, limits): + if self.error: + raise self.error + self.called_with_args.append((resource, limits)) + + def __getattr__(self, name): + # I only over-ride portions of the resource module + try: + return object.__getattr__(self, name) + except AttributeError: + return getattr(resource, name) + + _orig_resource = manager.resource + _orig_environ = os.environ + try: + manager.resource = MockResource() + manager.os.environ = {} + manager.setup_env() + expected = [ + (resource.RLIMIT_NOFILE, (manager.MAX_DESCRIPTORS, + manager.MAX_DESCRIPTORS)), + (resource.RLIMIT_DATA, (manager.MAX_MEMORY, + manager.MAX_MEMORY)), + ] + self.assertEquals(manager.resource.called_with_args, expected) + self.assertEquals(manager.os.environ['PYTHON_EGG_CACHE'], '/tmp') + + # test error condition + manager.resource = MockResource(error=ValueError()) + manager.os.environ = {} + manager.setup_env() + self.assertEquals(manager.resource.called_with_args, []) + self.assertEquals(manager.os.environ['PYTHON_EGG_CACHE'], '/tmp') + + manager.resource = MockResource(error=OSError()) + manager.os.environ = {} + self.assertRaises(OSError, manager.setup_env) + self.assertEquals(manager.os.environ.get('PYTHON_EGG_CACHE'), None) + finally: + manager.resource = _orig_resource + os.environ = _orig_environ + + def test_command_wrapper(self): @manager.command @@ -1003,45 +1050,57 @@ class TestServer(unittest.TestCase): class TestManager(unittest.TestCase): def test_create(self): - controller = manager.Manager(['test']) - self.assertEquals(len(controller.servers), 1) - server = controller.servers.pop() + m = manager.Manager(['test']) + self.assertEquals(len(m.servers), 1) + server = m.servers.pop() self.assert_(isinstance(server, manager.Server)) self.assertEquals(server.server, 'test-server') # test multi-server and simple dedupe servers = ['object-replicator', 'object-auditor', 'object-replicator'] - controller = manager.Manager(servers) - self.assertEquals(len(controller.servers), 2) - for server in controller.servers: + m = manager.Manager(servers) + self.assertEquals(len(m.servers), 2) + for server in m.servers: self.assert_(server.server in servers) # test all - controller = manager.Manager(['all']) - self.assertEquals(len(controller.servers), len(manager.ALL_SERVERS)) - for server in controller.servers: + m = manager.Manager(['all']) + self.assertEquals(len(m.servers), len(manager.ALL_SERVERS)) + for server in m.servers: self.assert_(server.server in manager.ALL_SERVERS) # test main - controller = manager.Manager(['main']) - self.assertEquals(len(controller.servers), len(manager.MAIN_SERVERS)) - for server in controller.servers: + m = manager.Manager(['main']) + self.assertEquals(len(m.servers), len(manager.MAIN_SERVERS)) + for server in m.servers: self.assert_(server.server in manager.MAIN_SERVERS) # test rest - controller = manager.Manager(['rest']) - self.assertEquals(len(controller.servers), len(manager.REST_SERVERS)) - for server in controller.servers: + m = manager.Manager(['rest']) + self.assertEquals(len(m.servers), len(manager.REST_SERVERS)) + for server in m.servers: self.assert_(server.server in manager.REST_SERVERS) # test main + rest == all - controller = manager.Manager(['main', 'rest']) - self.assertEquals(len(controller.servers), len(manager.ALL_SERVERS)) - for server in controller.servers: + m = manager.Manager(['main', 'rest']) + self.assertEquals(len(m.servers), len(manager.ALL_SERVERS)) + for server in m.servers: self.assert_(server.server in manager.ALL_SERVERS) # test dedupe - controller = manager.Manager(['main', 'rest', 'proxy', 'object', + m = manager.Manager(['main', 'rest', 'proxy', 'object', 'container', 'account']) - self.assertEquals(len(controller.servers), len(manager.ALL_SERVERS)) - for server in controller.servers: + self.assertEquals(len(m.servers), len(manager.ALL_SERVERS)) + for server in m.servers: self.assert_(server.server in manager.ALL_SERVERS) + # test glob + m = manager.Manager(['object-*']) + object_servers = [s for s in manager.ALL_SERVERS if + s.startswith('object')] + self.assertEquals(len(m.servers), len(object_servers)) + for s in m.servers: + self.assert_(str(s) in object_servers) + m = manager.Manager(['*-replicator']) + replicators = [s for s in manager.ALL_SERVERS if + s.endswith('replicator')] + for s in m.servers: + self.assert_(str(s) in replicators) - #TODO: more tests + def test_watch_server_pids(self): class MockOs(): WNOHANG = os.WNOHANG @@ -1081,8 +1140,9 @@ class TestManager(unittest.TestCase): class MockServer(): def __init__(self, server): self.server = server - def get_running_pids(): - pass + + def get_running_pids(self): + return {} _orig_os = manager.os _orig_time = manager.time @@ -1091,18 +1151,19 @@ class TestManager(unittest.TestCase): manager.os = MockOs() manager.time = MockTime() manager.Server = MockServer - controller = manager.Manager(['test']) - self.assertEquals(len(controller.servers), 1) - server = controller.servers.pop() + m = manager.Manager(['test']) + self.assertEquals(len(m.servers), 1) + server = m.servers.pop() server_pids = { server: [1], } # test default interval - gen = controller.watch_server_pids(server_pids) + gen = m.watch_server_pids(server_pids) self.assertEquals([x for x in gen], []) # test small interval - gen = controller.watch_server_pids(server_pids, interval=1) - # self.assertEquals([x for x in gen], []) + gen = m.watch_server_pids(server_pids, interval=1) + # TODO: More tests! + #self.assertEquals([x for x in gen], []) finally: manager.os = _orig_os manager.time = _orig_time @@ -1136,18 +1197,18 @@ class TestManager(unittest.TestCase): old_server_class = manager.Server try: manager.Server = MockServer - controller = manager.Manager(['test']) - status = controller.status() + m = manager.Manager(['test']) + status = m.status() self.assertEquals(status, 0) - controller = manager.Manager(['error']) - status = controller.status() + m = manager.Manager(['error']) + status = m.status() self.assertEquals(status, 1) # test multi-server - controller = manager.Manager(['test', 'error']) + m = manager.Manager(['test', 'error']) kwargs = {'key': 'value'} - status = controller.status(**kwargs) + status = m.status(**kwargs) self.assertEquals(status, 1) - for server in controller.servers: + for server in m.servers: self.assertEquals(server.called_kwargs, [kwargs]) finally: manager.Server = old_server_class @@ -1191,27 +1252,27 @@ class TestManager(unittest.TestCase): manager.Server = MockServer # test no errors on launch - controller = manager.Manager(['proxy', 'error']) - status = controller.start() + m = manager.Manager(['proxy', 'error']) + status = m.start() self.assertEquals(status, 0) - for server in controller.servers: + for server in m.servers: self.assertEquals(server.called['launch'], [{}]) # test error on wait - controller = manager.Manager(['proxy', 'error']) + m = manager.Manager(['proxy', 'error']) kwargs = {'wait': True} - status = controller.start(**kwargs) + status = m.start(**kwargs) self.assertEquals(status, 1) - for server in controller.servers: + for server in m.servers: self.assertEquals(server.called['launch'], [kwargs]) self.assertEquals(server.called['wait'], [kwargs]) # test interact - controller = manager.Manager(['proxy', 'error']) + m = manager.Manager(['proxy', 'error']) kwargs = {'daemon': False} - status = controller.start(**kwargs) + status = m.start(**kwargs) self.assertEquals(status, 1) - for server in controller.servers: + for server in m.servers: self.assertEquals(server.called['launch'], [kwargs]) self.assertEquals(server.called['interact'], [kwargs]) finally: From da27b3b7503bea823bd442f7d93eeed57d427932 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 11 Feb 2011 17:39:44 -0800 Subject: [PATCH 46/64] Make swift-auth-to-swauth work with really old devauth dbs; update swauth to accept non-alnum chars in account and user names. --- bin/swift-auth-to-swauth | 26 +++++++++------------- swift/common/middleware/swauth.py | 21 ++++++++--------- test/unit/common/middleware/test_swauth.py | 17 ++++++++++++++ 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/bin/swift-auth-to-swauth b/bin/swift-auth-to-swauth index 93cb4fe199..e1010c315a 100755 --- a/bin/swift-auth-to-swauth +++ b/bin/swift-auth-to-swauth @@ -23,16 +23,18 @@ import sqlite3 if __name__ == '__main__': gettext.install('swift', unicode=1) - if len(argv) != 4 or argv[1] != '-K': - exit('Syntax: %s -K ' % argv[0]) - _junk, _junk, super_admin_key, auth_db = argv - # This version will not attempt to prep swauth - # call(['swauth-prep', '-K', super_admin_key]) + if len(argv) != 2: + exit('Syntax: %s ' % argv[0]) + _junk, auth_db = argv conn = sqlite3.connect(auth_db) - for account, cfaccount, user, password, admin, reseller_admin in \ - conn.execute('SELECT account, cfaccount, user, password, admin, ' - 'reseller_admin FROM account'): - cmd = ['swauth-add-user', '-K', super_admin_key, '-s', + try: + listing = conn.execute('SELECT account, cfaccount, user, password, ' + 'admin, reseller_admin FROM account') + except sqlite3.OperationalError, err: + listing = conn.execute('SELECT account, cfaccount, user, password, ' + '"f", "f" FROM account') + for account, cfaccount, user, password, admin, reseller_admin in listing: + cmd = ['swauth-add-user', '-K', '', '-s', cfaccount.split('_', 1)[1]] if admin == 't': cmd.append('-a') @@ -40,9 +42,3 @@ if __name__ == '__main__': cmd.append('-r') cmd.extend([account, user, password]) print ' '.join(cmd) - # For this version, the script will only print out the commands - # call(cmd) - print '----------------------------------------------------------------' - print ' Assuming the above worked perfectly, you should copy and paste ' - print ' those lines into your ~/bin/recreateaccounts script.' - print '----------------------------------------------------------------' diff --git a/swift/common/middleware/swauth.py b/swift/common/middleware/swauth.py index 9d585c0a6b..68b0d7afaf 100644 --- a/swift/common/middleware/swauth.py +++ b/swift/common/middleware/swauth.py @@ -268,7 +268,7 @@ class Swauth(object): user_groups = (req.remote_user or '').split(',') if '.reseller_admin' in user_groups and \ account != self.reseller_prefix and \ - account[len(self.reseller_prefix)].isalnum(): + account[len(self.reseller_prefix)] != '.': return None if account in user_groups and \ (req.method not in ('DELETE', 'PUT') or container): @@ -474,7 +474,7 @@ class Swauth(object): explained above. """ account = req.path_info_pop() - if req.path_info or not account.isalnum(): + if req.path_info or not account or account[0] == '.': return HTTPBadRequest(request=req) if not self.is_account_admin(req, account): return HTTPForbidden(request=req) @@ -550,7 +550,7 @@ class Swauth(object): if not self.is_reseller_admin(req): return HTTPForbidden(request=req) account = req.path_info_pop() - if req.path_info != '/.services' or not account.isalnum(): + if req.path_info != '/.services' or not account or account[0] == '.': return HTTPBadRequest(request=req) try: new_services = json.loads(req.body) @@ -596,7 +596,7 @@ class Swauth(object): if not self.is_reseller_admin(req): return HTTPForbidden(request=req) account = req.path_info_pop() - if req.path_info or not account.isalnum(): + if req.path_info or not account or account[0] == '.': return HTTPBadRequest(request=req) # Ensure the container in the main auth account exists (this # container represents the new account) @@ -678,7 +678,7 @@ class Swauth(object): if not self.is_reseller_admin(req): return HTTPForbidden(request=req) account = req.path_info_pop() - if req.path_info or not account.isalnum(): + if req.path_info or not account or account[0] == '.': return HTTPBadRequest(request=req) # Make sure the account has no users and get the account_id marker = '' @@ -798,8 +798,8 @@ class Swauth(object): """ account = req.path_info_pop() user = req.path_info_pop() - if req.path_info or not account.isalnum() or \ - (not user.isalnum() and user != '.groups'): + if req.path_info or not account or account[0] == '.' or not user or \ + (user[0] == '.' and user != '.groups'): return HTTPBadRequest(request=req) if not self.is_account_admin(req, account): return HTTPForbidden(request=req) @@ -873,8 +873,8 @@ class Swauth(object): req.headers.get('x-auth-user-reseller-admin') == 'true' if reseller_admin: admin = True - if req.path_info or not account.isalnum() or not user.isalnum() or \ - not key: + if req.path_info or not account or account[0] == '.' or not user or \ + user[0] == '.' or not key: return HTTPBadRequest(request=req) if reseller_admin: if not self.is_super_admin(req): @@ -922,7 +922,8 @@ class Swauth(object): # Validate path info account = req.path_info_pop() user = req.path_info_pop() - if req.path_info or not account.isalnum() or not user.isalnum(): + if req.path_info or not account or account[0] == '.' or not user or \ + user[0] == '.': return HTTPBadRequest(request=req) if not self.is_account_admin(req, account): return HTTPForbidden(request=req) diff --git a/test/unit/common/middleware/test_swauth.py b/test/unit/common/middleware/test_swauth.py index ce3681ac06..eeda4f0cbf 100644 --- a/test/unit/common/middleware/test_swauth.py +++ b/test/unit/common/middleware/test_swauth.py @@ -2576,6 +2576,23 @@ class TestAuth(unittest.TestCase): {"groups": [{"name": "act:usr"}, {"name": "act"}], "auth": "plaintext:key"}) + def test_put_user_special_chars_success(self): + self.test_auth.app = FakeApp(iter([ + ('200 Ok', {'X-Container-Meta-Account-Id': 'AUTH_cfa'}, ''), + # PUT of user object + ('201 Created', {}, '')])) + resp = Request.blank('/auth/v2/act/u_s-r', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Auth-Admin-User': '.super_admin', + 'X-Auth-Admin-Key': 'supertest', + 'X-Auth-User-Key': 'key'} + ).get_response(self.test_auth) + self.assertEquals(resp.status_int, 201) + self.assertEquals(self.test_auth.app.calls, 2) + self.assertEquals(json.loads(self.test_auth.app.request.body), + {"groups": [{"name": "act:u_s-r"}, {"name": "act"}], + "auth": "plaintext:key"}) + def test_put_user_account_admin_success(self): self.test_auth.app = FakeApp(iter([ ('200 Ok', {'X-Container-Meta-Account-Id': 'AUTH_cfa'}, ''), From 8a66319f14ca6347c84ecbb14e21b5cf6b4b7d31 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Sat, 12 Feb 2011 01:49:24 -0600 Subject: [PATCH 47/64] more test, swift.common.manager coverage > 90% --- swift/common/manager.py | 88 ++++----- test/unit/common/test_manager.py | 307 ++++++++++++++++++++----------- 2 files changed, 247 insertions(+), 148 deletions(-) diff --git a/swift/common/manager.py b/swift/common/manager.py index 089895044e..8e78ff4428 100644 --- a/swift/common/manager.py +++ b/swift/common/manager.py @@ -43,6 +43,7 @@ KILL_WAIT = 15 # seconds to wait for servers to die MAX_DESCRIPTORS = 32768 MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB + def setup_env(): """Try to increase resource limits of the OS. Move PYTHON_EGG_CACHE to /tmp """ @@ -58,6 +59,7 @@ def setup_env(): os.environ['PYTHON_EGG_CACHE'] = '/tmp' return + def command(func): """ Decorator to declare which methods are accessible as commands, commands @@ -74,6 +76,44 @@ def command(func): return wrapped +def watch_server_pids(server_pids, interval=1, **kwargs): + """Monitor a collection of server pids yeilding back those pids that + aren't responding to signals. + + :param server_pids: a dict, lists of pids [int,...] keyed on + Server objects + """ + status = {} + start = time.time() + end = start + interval + server_pids = dict(server_pids) # make a copy + while interval: + for server, pids in server_pids.items(): + for pid in pids: + try: + # let pid stop if it wants to + os.waitpid(pid, os.WNOHANG) + except OSError, e: + if e.errno not in (errno.ECHILD, errno.ESRCH): + raise # else no such child/process + # check running pids for server + status[server] = server.get_running_pids(**kwargs) + for pid in pids: + # original pids no longer in running pids! + if pid not in status[server]: + yield server, pid + # update active pids list using running_pids + server_pids[server] = status[server] + if not [p for server, pids in status.items() for p in pids]: + # no more running pids + break + if time.time() > end: + break + else: + time.sleep(0.1) + return + + class UnknownCommandError(Exception): pass @@ -105,42 +145,6 @@ class Manager(): for name in server_names: self.servers.add(Server(name)) - def watch_server_pids(self, server_pids, interval=0, **kwargs): - """Monitor a collection of server pids yeilding back those pids that - aren't responding to signals. - - :param server_pids: a dict, lists of pids [int,...] keyed on - Server objects - """ - status = {} - start = time.time() - end = start + interval - while interval: - for server, pids in server_pids.items(): - for pid in pids: - try: - # let pid stop if it wants to - os.waitpid(pid, os.WNOHANG) - except OSError, e: - if e.errno not in (errno.ECHILD, errno.ESRCH): - raise # else no such child/process - # check running pids for server - status[server] = server.get_running_pids(**kwargs) - for pid in pids: - # original pids no longer in running pids! - if pid not in status[server]: - yield server, pid - # update active pids list using running_pids - server_pids[server] = status[server] - if not [p for server, pids in status.items() for p in pids]: - # no more running pids - break - if time.time() > end: - break - else: - time.sleep(0.1) - return - @command def status(self, **kwargs): """display status of tracked pids for server @@ -206,11 +210,11 @@ class Manager(): server_pids[server] = signaled_pids # all signaled_pids, i.e. list(itertools.chain(*server_pids.values())) - signaled_pids = [p for server, pid in server_pids.items() for p in pid] + signaled_pids = [p for server, pids in server_pids.items() for p in pids] # keep track of the pids yeiled back as killed for all servers killed_pids = set() - for server, killed_pid in self.watch_server_pids(server_pids, - interval=KILL_WAIT, **kwargs): + for server, killed_pid in watch_server_pids(server_pids, + interval=KILL_WAIT, **kwargs): print "%s (%s) appears to have stopped" % (server, killed_pid) killed_pids.add(killed_pid) if not killed_pids.symmetric_difference(signaled_pids): @@ -230,7 +234,7 @@ class Manager(): """ kwargs['graceful'] = True status = 0 - self.stop(**kwargs) + status += self.stop(**kwargs) return status @command @@ -238,8 +242,8 @@ class Manager(): """stops then restarts server """ status = 0 - self.stop(**kwargs) - self.start(**kwargs) + status += self.stop(**kwargs) + status += self.start(**kwargs) return status @command diff --git a/test/unit/common/test_manager.py b/test/unit/common/test_manager.py index c48829d2c7..e385082cb8 100644 --- a/test/unit/common/test_manager.py +++ b/test/unit/common/test_manager.py @@ -31,6 +31,7 @@ from swift.common import manager DUMMY_SIG = 1 + class MockOs(): def __init__(self, pids): @@ -79,7 +80,7 @@ class TestManagerModule(unittest.TestCase): def __init__(self, error=None): self.error = error self.called_with_args = [] - + def setrlimit(self, resource, limits): if self.error: raise self.error @@ -121,8 +122,6 @@ class TestManagerModule(unittest.TestCase): finally: manager.resource = _orig_resource os.environ = _orig_environ - - def test_command_wrapper(self): @manager.command @@ -139,9 +138,122 @@ class TestManagerModule(unittest.TestCase): self.assert_(hasattr(myfunc, 'publicly_accessible')) self.assert_(myfunc.publicly_accessible) + def test_watch_server_pids(self): + class MockOs(): + WNOHANG = os.WNOHANG + + def __init__(self, pid_map={}): + self.pid_map = {} + for pid, v in pid_map.items(): + self.pid_map[pid] = (x for x in v) + + def waitpid(self, pid, options): + try: + rv = self.pid_map[pid].next() + except StopIteration: + raise OSError(errno.ECHILD, os.strerror(errno.ECHILD)) + except KeyError: + raise OSError(errno.ESRCH, os.strerror(errno.ESRCH)) + if isinstance(rv, Exception): + raise rv + else: + return rv + + class MockTime(): + def __init__(self, ticks=None): + self.tock = time() + if not ticks: + ticks = [] + + self.ticks = (t for t in ticks) + + def time(self): + try: + self.tock += self.ticks.next() + except StopIteration: + self.tock += 1 + return self.tock + + def sleep(*args): + return + + class MockServer(): + + def __init__(self, pids, zombie=0): + self.heartbeat = (pids for _ in range(zombie)) + + def get_running_pids(self): + try: + rv = self.heartbeat.next() + return rv + except StopIteration: + return {} + + _orig_os = manager.os + _orig_time = manager.time + _orig_server = manager.Server + try: + manager.time = MockTime() + manager.os = MockOs() + # this server always says it's dead when you ask for running pids + server = MockServer([1]) + # list of pids keyed on servers to watch + server_pids = { + server: [1], + } + # basic test, server dies + gen = manager.watch_server_pids(server_pids) + expected = [(server, 1)] + self.assertEquals([x for x in gen], [(server, 1)]) + # start long running server and short interval + server = MockServer([1], zombie=15) + server_pids = { + server: [1], + } + gen = manager.watch_server_pids(server_pids) + self.assertEquals([x for x in gen], []) + # wait a little longer + gen = manager.watch_server_pids(server_pids, interval=15) + self.assertEquals([x for x in gen], [(server, 1)]) + # zombie process + server = MockServer([1], zombie=200) + server_pids = { + server: [1], + } + # test weird os error + manager.os = MockOs({1: [OSError()]}) + gen = manager.watch_server_pids(server_pids) + self.assertRaises(OSError, lambda: [x for x in gen]) + # test multi-server + server1 = MockServer([1, 10], zombie=200) + server2 = MockServer([2, 20], zombie=8) + server_pids = { + server1: [1, 10], + server2: [2, 20], + } + pid_map = { + 1: [None for _ in range(10)], + 2: [None for _ in range(8)], + 20: [None for _ in range(4)], + } + manager.os = MockOs(pid_map) + gen = manager.watch_server_pids(server_pids, + interval=manager.KILL_WAIT) + expected = [ + (server2, 2), + (server2, 20), + ] + self.assertEquals([x for x in gen], expected) + + finally: + manager.os = _orig_os + manager.time = _orig_time + manager.Server = _orig_server + def test_exc(self): self.assert_(issubclass(manager.UnknownCommandError, Exception)) + class TestServer(unittest.TestCase): def tearDown(self): @@ -876,7 +988,7 @@ class TestServer(unittest.TestCase): procs.append(MockProcess(fail=fail)) server.procs = procs self.assert_(server.interact() > 0) - + def test_launch(self): # stubs ini_files = ( @@ -987,8 +1099,6 @@ class TestServer(unittest.TestCase): 'number': 4 } self.assertEquals(mock_spawn.kwargs, [expected]) - - finally: sys.stdout = old_stdout @@ -1047,6 +1157,7 @@ class TestServer(unittest.TestCase): self.assertFalse(os.path.exists(conf4)) self.assertFalse(os.path.exists(conf3)) + class TestManager(unittest.TestCase): def test_create(self): @@ -1100,93 +1211,13 @@ class TestManager(unittest.TestCase): for s in m.servers: self.assert_(str(s) in replicators) - - def test_watch_server_pids(self): - class MockOs(): - WNOHANG = os.WNOHANG - def __init__(self, pid_map={}): - self.pid_map = {} - for pid,v in pid_map.items(): - self.pid_map[pid] = (x for x in v) - def waitpid(self, pid, options): - try: - rv = self.pid_map[pid].next() - except StopIteration: - raise OSError(errno.ECHILD, os.strerror(errno.ECHILD)) - except KeyError: - raise OSError(errno.ESRCH, os.strerror(errno.ESRCH)) - if isinstance(rv, Exception): - raise rv() - else: - return rv - - class MockTime(): - def __init__(self, ticks=None): - self.tock = time() - if ticks: - ticks = [t for t in ticks] - else: - ticks = [] - - self.ticks = (t for t in ticks) - - def time(self): - try: - self.tock += self.ticks.next() - except StopIteration: - self.tock += 1 - return self.tock - - class MockServer(): - def __init__(self, server): - self.server = server - - def get_running_pids(self): - return {} - - _orig_os = manager.os - _orig_time = manager.time - _orig_server= manager.Server - try: - manager.os = MockOs() - manager.time = MockTime() - manager.Server = MockServer - m = manager.Manager(['test']) - self.assertEquals(len(m.servers), 1) - server = m.servers.pop() - server_pids = { - server: [1], - } - # test default interval - gen = m.watch_server_pids(server_pids) - self.assertEquals([x for x in gen], []) - # test small interval - gen = m.watch_server_pids(server_pids, interval=1) - # TODO: More tests! - #self.assertEquals([x for x in gen], []) - finally: - manager.os = _orig_os - manager.time = _orig_time - manager.Server = _orig_server - - def test_get_command(self): - raise SkipTest - - def test_list_commands(self): - for cmd, help in manager.Manager.list_commands(): - method = getattr(manager.Manager, cmd.replace('-', '_'), None) - self.assert_(method, '%s is not a command' % cmd) - self.assert_(getattr(method, 'publicly_accessible', False)) - self.assertEquals(method.__doc__.strip(), help) - - def test_run_command(self): - raise SkipTest - def test_status(self): class MockServer(): + def __init__(self, server): self.server = server self.called_kwargs = [] + def status(self, **kwargs): self.called_kwargs.append(kwargs) if 'error' in self.server: @@ -1216,8 +1247,8 @@ class TestManager(unittest.TestCase): def test_start(self): def mock_setup_env(): getattr(mock_setup_env, 'called', []).append(True) - class MockServer(): + class MockServer(): def __init__(self, server): self.server = server self.called = defaultdict(list) @@ -1231,16 +1262,15 @@ class TestManager(unittest.TestCase): return 1 else: return 0 - + + def stop(self, **kwargs): + self.called['stop'].append(kwargs) + def interact(self, **kwargs): self.called['interact'].append(kwargs) - # TODO: test user quit - """ if 'raise' in self.server: raise KeyboardInterrupt - el - """ - if 'error' in self.server: + elif 'error' in self.server: return 1 else: return 0 @@ -1266,7 +1296,7 @@ class TestManager(unittest.TestCase): for server in m.servers: self.assertEquals(server.called['launch'], [kwargs]) self.assertEquals(server.called['wait'], [kwargs]) - + # test interact m = manager.Manager(['proxy', 'error']) kwargs = {'daemon': False} @@ -1275,11 +1305,14 @@ class TestManager(unittest.TestCase): for server in m.servers: self.assertEquals(server.called['launch'], [kwargs]) self.assertEquals(server.called['interact'], [kwargs]) + m = manager.Manager(['raise']) + kwargs = {'daemon': False} + status = m.start(**kwargs) + finally: manager.setup_env = old_setup_env manager.Server = old_swift_server - def test_wait(self): class MockServer(): def __init__(self, server): @@ -1341,7 +1374,6 @@ class TestManager(unittest.TestCase): self.assert_(called_kwargs['once']) finally: manager.Server = orig_swift_server - def test_no_daemon(self): class MockServer(): @@ -1387,7 +1419,6 @@ class TestManager(unittest.TestCase): def launch(self, **kwargs): return self.called['launch'].append(kwargs) - orig_swift_server = manager.Server try: @@ -1408,25 +1439,89 @@ class TestManager(unittest.TestCase): self.assertEquals(len(server.called['interact']), 0) finally: manager.Server = orig_swift_server - def test_stop(self): - raise SkipTest + class MockServerFactory(): + class MockServer(): + def __init__(self, pids): + self.pids = pids + def stop(self, **kwargs): + return self.pids + + def __init__(self, server_pids): + self.server_pids = server_pids + + def __call__(self, server): + return MockServerFactory.MockServer(self.server_pids[server]) + + + def mock_watch_server_pids(server_pids, **kwargs): + for server, pids in server_pids.items(): + for pid in pids: + if pid is None: + continue + yield server, pid + + _orig_server = manager.Server + _orig_watch_server_pids = manager.watch_server_pids + try: + manager.watch_server_pids = mock_watch_server_pids + # test stop one server + server_pids = { + 'test': [1] + } + manager.Server = MockServerFactory(server_pids) + m = manager.Manager(['test']) + status = m.stop() + self.assertEquals(status, 0) + # test not running + server_pids = { + 'test': [] + } + manager.Server = MockServerFactory(server_pids) + m = manager.Manager(['test']) + status = m.stop() + self.assertEquals(status, 1) + # test won't die + server_pids = { + 'test': [None] + } + manager.Server = MockServerFactory(server_pids) + m = manager.Manager(['test']) + status = m.stop() + self.assertEquals(status, 1) + + finally: + manager.Server = _orig_server + manager.watch_server_pids = _orig_watch_server_pids + + # TODO: more tests def test_shutdown(self): - raise SkipTest + pass def test_restart(self): - raise SkipTest + pass def test_reload(self): - raise SkipTest + pass def test_force_reload(self): - raise SkipTest + pass + def test_get_command(self): + pass + + def test_list_commands(self): + for cmd, help in manager.Manager.list_commands(): + method = getattr(manager.Manager, cmd.replace('-', '_'), None) + self.assert_(method, '%s is not a command' % cmd) + self.assert_(getattr(method, 'publicly_accessible', False)) + self.assertEquals(method.__doc__.strip(), help) + + def test_run_command(self): + pass if __name__ == '__main__': unittest.main() - From bfae3625ef4f2fb1865dcc1dc3ba07e29f2a4d7c Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Sat, 12 Feb 2011 02:22:01 -0600 Subject: [PATCH 48/64] fixed license dates --- bin/swift-init | 2 +- swift/common/manager.py | 2 +- test/unit/common/test_manager.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/swift-init b/bin/swift-init index ff7cc80285..c6e770bd8f 100644 --- a/bin/swift-init +++ b/bin/swift-init @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2010 OpenStack, LLC. +# Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/swift/common/manager.py b/swift/common/manager.py index 60e1d9527f..b5082d90af 100644 --- a/swift/common/manager.py +++ b/swift/common/manager.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010 OpenStack, LLC. +# Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/test/unit/common/test_manager.py b/test/unit/common/test_manager.py index e385082cb8..ebdaf3f647 100644 --- a/test/unit/common/test_manager.py +++ b/test/unit/common/test_manager.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010 OpenStack, LLC. +# Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From c03a302f7c6f5518661f7750b0b4896510cb3120 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Sat, 12 Feb 2011 02:57:33 -0600 Subject: [PATCH 49/64] PEP8 --- swift/common/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index 2bb751ebf7..e185e864b5 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -80,7 +80,6 @@ def validate_configuration(): print "Error: [swift-hash]: swift_hash_path_suffix missing from " \ "/etc/swift/swift.conf" sys.exit(1) - def load_libc_function(func_name): @@ -474,7 +473,10 @@ def capture_stdio(logger, **kwargs): stdio_fds = [0, 1, 2] for _junk, handler in getattr(get_logger, 'console_handler4logger', {}).items(): - stdio_fds.remove(handler.stream.fileno()) + try: + stdio_fds.remove(handler.stream.fileno()) + except ValueError: + pass # fd not in list with open(os.devnull, 'r+b') as nullfile: # close stdio (excludes fds open for logging) @@ -841,6 +843,7 @@ def remove_file(path): except OSError: pass + def audit_location_generator(devices, datadir, mount_check=True, logger=None): ''' Given a devices path and a data directory, yield (path, device, From 51064d31ebb28f13e6f3d8164439ae747b801766 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Sat, 12 Feb 2011 03:25:29 -0600 Subject: [PATCH 50/64] renamed ini_files to conf_files in code --- swift/common/manager.py | 78 +++++++++---------- test/unit/common/test_manager.py | 124 +++++++++++++++---------------- 2 files changed, 101 insertions(+), 101 deletions(-) diff --git a/swift/common/manager.py b/swift/common/manager.py index b5082d90af..d2a8c3f0d0 100644 --- a/swift/common/manager.py +++ b/swift/common/manager.py @@ -331,25 +331,25 @@ class Server(): except AttributeError: return False - def get_pid_file_name(self, ini_file): - """Translate ini_file to a corresponding pid_file + def get_pid_file_name(self, conf_file): + """Translate conf_file to a corresponding pid_file - :param ini_file: an ini_file for this server, a string + :param conf_file: an conf_file for this server, a string - :returns: the pid_file for this ini_file + :returns: the pid_file for this conf_file """ - return ini_file.replace( + return conf_file.replace( os.path.normpath(SWIFT_DIR), RUN_DIR, 1).replace( '%s-server' % self.type, self.server, 1).rsplit( '.conf', 1)[0] + '.pid' - def get_ini_file_name(self, pid_file): - """Translate pid_file to a corresponding ini_file + def get_conf_file_name(self, pid_file): + """Translate pid_file to a corresponding conf_file :param pid_file: a pid_file for this server, a string - :returns: the ini_file for this pid_file + :returns: the conf_file for this pid_file """ return pid_file.replace( @@ -357,35 +357,35 @@ class Server(): self.server, '%s-server' % self.type, 1).rsplit( '.pid', 1)[0] + '.conf' - def ini_files(self, **kwargs): + def conf_files(self, **kwargs): """Get ini files for this server :param: number, if supplied will only lookup the nth server :returns: list of ini files """ - found_ini_files = search_tree(SWIFT_DIR, '%s-server*' % self.type, + found_conf_files = search_tree(SWIFT_DIR, '%s-server*' % self.type, '.conf') number = kwargs.get('number') if number: try: - ini_files = [found_ini_files[number - 1]] + conf_files = [found_conf_files[number - 1]] except IndexError: - ini_files = [] + conf_files = [] else: - ini_files = found_ini_files - if not ini_files: + conf_files = found_conf_files + if not conf_files: # maybe there's a config file(s) out there, but I couldn't find it! if not kwargs.get('quiet'): print('Unable to locate config %sfor %s' % ( ('number %s ' % number if number else ''), self.server)) if kwargs.get('verbose') and not kwargs.get('quiet'): - if found_ini_files: + if found_conf_files: print('Found configs:') - for i, ini_file in enumerate(found_ini_files): - print(' %d) %s' % (i + 1, ini_file)) + for i, conf_file in enumerate(found_conf_files): + print(' %d) %s' % (i + 1, conf_file)) - return ini_files + return conf_files def pid_files(self, **kwargs): """Get pid files for this server @@ -397,11 +397,11 @@ class Server(): pid_files = search_tree(RUN_DIR, '%s*' % self.server, '.pid') number = kwargs.get('number', 0) if number: - ini_files = self.ini_files(**kwargs) - # limt pid_files the one who translates to the indexed ini_file for + conf_files = self.conf_files(**kwargs) + # limt pid_files the one who translates to the indexed conf_file for # this given number pid_files = [pid_file for pid_file in pid_files if - self.get_ini_file_name(pid_file) in ini_files] + self.get_conf_file_name(pid_file) in conf_files] return pid_files def iter_pid_files(self, **kwargs): @@ -474,29 +474,29 @@ class Server(): number = kwargs.get('number', 0) if number: kwargs['quiet'] = True - ini_files = self.ini_files(**kwargs) - if ini_files: + conf_files = self.conf_files(**kwargs) + if conf_files: print "%s #%d not running (%s)" % (self.server, number, - ini_files[0]) + conf_files[0]) else: print "No %s running" % self.server return 1 for pid, pid_file in pids.items(): - ini_file = self.get_ini_file_name(pid_file) - print "%s running (%s - %s)" % (self.server, pid, ini_file) + conf_file = self.get_conf_file_name(pid_file) + print "%s running (%s - %s)" % (self.server, pid, conf_file) return 0 - def spawn(self, ini_file, once=False, wait=False, daemon=True, **kwargs): + def spawn(self, conf_file, once=False, wait=False, daemon=True, **kwargs): """Launch a subprocess for this server. - :param ini_file: path to ini_file to use as first arg + :param conf_file: path to conf_file to use as first arg :param once: boolean, add once argument to command :param wait: boolean, if true capture stdout with a pipe :param daemon: boolean, if true ask server to log to console :returns : the pid of the spawned process """ - args = [self.cmd, ini_file] + args = [self.cmd, conf_file] if once: args.append('once') if not daemon: @@ -516,7 +516,7 @@ class Server(): else: re_out = open(os.devnull, 'w+b') proc = subprocess.Popen(args, stdout=re_out, stderr=re_err) - pid_file = self.get_pid_file_name(ini_file) + pid_file = self.get_pid_file_name(conf_file) write_file(pid_file, proc.pid) self.procs.append(proc) return proc.pid @@ -551,22 +551,22 @@ class Server(): """ Collect ini files and attempt to spawn the processes for this server """ - ini_files = self.ini_files(**kwargs) - if not ini_files: + conf_files = self.conf_files(**kwargs) + if not conf_files: return [] pids = self.get_running_pids(**kwargs) already_started = False for pid, pid_file in pids.items(): - ini_file = self.get_ini_file_name(pid_file) + conf_file = self.get_conf_file_name(pid_file) # for legacy compat you can't start other servers if one server is # already running (unless -n specifies which one you want), this # restriction could potentially be lifted, and launch could start # any unstarted instances - if ini_file in ini_files: + if conf_file in conf_files: already_started = True - print "%s running (%s - %s)" % (self.server, pid, ini_file) + print "%s running (%s - %s)" % (self.server, pid, conf_file) elif not kwargs.get('number', 0): already_started = True print "%s running (%s - %s)" % (self.server, pid, pid_file) @@ -582,20 +582,20 @@ class Server(): # TODO: check if self.cmd exists? pids = {} - for ini_file in ini_files: + for conf_file in conf_files: if kwargs.get('once'): msg = 'Running %s once' % self.server else: msg = 'Starting %s' % self.server - print '%s...(%s)' % (msg, ini_file) + print '%s...(%s)' % (msg, conf_file) try: - pid = self.spawn(ini_file, **kwargs) + pid = self.spawn(conf_file, **kwargs) except OSError, e: if e.errno == errno.ENOENT: # cmd does not exist print "%s does not exist" % self.cmd break - pids[pid] = ini_file + pids[pid] = conf_file return pids diff --git a/test/unit/common/test_manager.py b/test/unit/common/test_manager.py index ebdaf3f647..de39a13cb7 100644 --- a/test/unit/common/test_manager.py +++ b/test/unit/common/test_manager.py @@ -295,54 +295,54 @@ class TestServer(unittest.TestCase): def test_get_pid_file_name(self): server = manager.Server('proxy') - ini_file = self.join_swift_dir('proxy-server.conf') + conf_file = self.join_swift_dir('proxy-server.conf') pid_file = self.join_run_dir('proxy-server.pid') - self.assertEquals(pid_file, server.get_pid_file_name(ini_file)) + self.assertEquals(pid_file, server.get_pid_file_name(conf_file)) server = manager.Server('object-replicator') - ini_file = self.join_swift_dir('object-server/1.conf') + conf_file = self.join_swift_dir('object-server/1.conf') pid_file = self.join_run_dir('object-replicator/1.pid') - self.assertEquals(pid_file, server.get_pid_file_name(ini_file)) + self.assertEquals(pid_file, server.get_pid_file_name(conf_file)) server = manager.Server('container-auditor') - ini_file = self.join_swift_dir( + conf_file = self.join_swift_dir( 'container-server/1/container-auditor.conf') pid_file = self.join_run_dir( 'container-auditor/1/container-auditor.pid') - self.assertEquals(pid_file, server.get_pid_file_name(ini_file)) + self.assertEquals(pid_file, server.get_pid_file_name(conf_file)) - def test_get_ini_file_name(self): + def test_get_conf_file_name(self): server = manager.Server('proxy') - ini_file = self.join_swift_dir('proxy-server.conf') + conf_file = self.join_swift_dir('proxy-server.conf') pid_file = self.join_run_dir('proxy-server.pid') - self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) + self.assertEquals(conf_file, server.get_conf_file_name(pid_file)) server = manager.Server('object-replicator') - ini_file = self.join_swift_dir('object-server/1.conf') + conf_file = self.join_swift_dir('object-server/1.conf') pid_file = self.join_run_dir('object-replicator/1.pid') - self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) + self.assertEquals(conf_file, server.get_conf_file_name(pid_file)) server = manager.Server('container-auditor') - ini_file = self.join_swift_dir( + conf_file = self.join_swift_dir( 'container-server/1/container-auditor.conf') pid_file = self.join_run_dir( 'container-auditor/1/container-auditor.pid') - self.assertEquals(ini_file, server.get_ini_file_name(pid_file)) + self.assertEquals(conf_file, server.get_conf_file_name(pid_file)) - def test_ini_files(self): + def test_conf_files(self): # test get single ini file - ini_files = ( + conf_files = ( 'proxy-server.conf', 'proxy-server.ini', 'auth-server.conf', ) - with temptree(ini_files) as t: + with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('proxy') - ini_files = server.ini_files() - self.assertEquals(len(ini_files), 1) - ini_file = ini_files[0] + conf_files = server.conf_files() + self.assertEquals(len(conf_files), 1) + conf_file = conf_files[0] proxy_conf = self.join_swift_dir('proxy-server.conf') - self.assertEquals(ini_file, proxy_conf) + self.assertEquals(conf_file, proxy_conf) # test multi server conf files & grouping of server-type config - ini_files = ( + conf_files = ( 'object-server1.conf', 'object-server/2.conf', 'object-server/object3.conf', @@ -350,46 +350,46 @@ class TestServer(unittest.TestCase): 'object-server.txt', 'proxy-server.conf', ) - with temptree(ini_files) as t: + with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('object-replicator') - ini_files = server.ini_files() - self.assertEquals(len(ini_files), 4) + conf_files = server.conf_files() + self.assertEquals(len(conf_files), 4) c1 = self.join_swift_dir('object-server1.conf') c2 = self.join_swift_dir('object-server/2.conf') c3 = self.join_swift_dir('object-server/object3.conf') c4 = self.join_swift_dir('object-server/conf/server4.conf') for c in [c1, c2, c3, c4]: - self.assert_(c in ini_files) + self.assert_(c in conf_files) # test configs returned sorted sorted_confs = sorted([c1, c2, c3, c4]) - self.assertEquals(ini_files, sorted_confs) + self.assertEquals(conf_files, sorted_confs) # test get single numbered conf - ini_files = ( + conf_files = ( 'account-server/1.conf', 'account-server/2.conf', 'account-server/3.conf', 'account-server/4.conf', ) - with temptree(ini_files) as t: + with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('account') - ini_files = server.ini_files(number=2) - self.assertEquals(len(ini_files), 1) - ini_file = ini_files[0] - self.assertEquals(ini_file, + conf_files = server.conf_files(number=2) + self.assertEquals(len(conf_files), 1) + conf_file = conf_files[0] + self.assertEquals(conf_file, self.join_swift_dir('account-server/2.conf')) # test missing config number - ini_files = server.ini_files(number=5) - self.assertFalse(ini_files) + conf_files = server.conf_files(number=5) + self.assertFalse(conf_files) # test verbose & quiet - ini_files = ( + conf_files = ( 'auth-server.ini', 'container-server/1.conf', ) - with temptree(ini_files) as t: + with temptree(conf_files) as t: manager.SWIFT_DIR = t old_stdout = sys.stdout try: @@ -397,21 +397,21 @@ class TestServer(unittest.TestCase): sys.stdout = f server = manager.Server('auth') # check warn "unable to locate" - ini_files = server.ini_files() - self.assertFalse(ini_files) + conf_files = server.conf_files() + self.assertFalse(conf_files) self.assert_('unable to locate' in pop_stream(f).lower()) # check quiet will silence warning - ini_files = server.ini_files(verbose=True, quiet=True) + conf_files = server.conf_files(verbose=True, quiet=True) self.assertEquals(pop_stream(f), '') # check found config no warning server = manager.Server('container-auditor') - ini_files = server.ini_files() + conf_files = server.conf_files() self.assertEquals(pop_stream(f), '') # check missing config number warn "unable to locate" - ini_files = server.ini_files(number=2) + conf_files = server.conf_files(number=2) self.assert_('unable to locate' in pop_stream(f).lower()) # check verbose lists configs - ini_files = server.ini_files(number=2, verbose=True) + conf_files = server.conf_files(number=2, verbose=True) c1 = self.join_swift_dir('container-server/1.conf') self.assert_(c1 in pop_stream(f)) finally: @@ -457,7 +457,7 @@ class TestServer(unittest.TestCase): self.assertEquals(pid_map, real_map) # test get pid_files by number - ini_files = ( + conf_files = ( 'object-server/1.conf', 'object-server/2.conf', 'object-server/3.conf', @@ -470,7 +470,7 @@ class TestServer(unittest.TestCase): ('object-server/5.pid', 5), ) - with temptree(ini_files) as swift_dir: + with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: @@ -667,7 +667,7 @@ class TestServer(unittest.TestCase): self.assert_(1 not in manager.os.pid_sigs) def test_status(self): - ini_files = ( + conf_files = ( 'test-server/1.conf', 'test-server/2.conf', 'test-server/3.conf', @@ -681,7 +681,7 @@ class TestServer(unittest.TestCase): ('test-server/4.pid', 4), ) - with temptree(ini_files) as swift_dir: + with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: @@ -706,7 +706,7 @@ class TestServer(unittest.TestCase): self.assertEquals(len(output), 1) line = output[0] self.assert_('test-server running' in line) - conf_four = self.join_swift_dir(ini_files[3]) + conf_four = self.join_swift_dir(conf_files[3]) self.assert_('4 - %s' % conf_four in line) # test some servers not running manager.os = MockOs([1, 2, 3]) @@ -722,7 +722,7 @@ class TestServer(unittest.TestCase): self.assertEquals(len(output), 1) line = output[0] self.assert_('not running' in line) - conf_three = self.join_swift_dir(ini_files[2]) + conf_three = self.join_swift_dir(conf_files[2]) self.assert_(conf_three in line) # test no running pids manager.os = MockOs([]) @@ -991,7 +991,7 @@ class TestServer(unittest.TestCase): def test_launch(self): # stubs - ini_files = ( + conf_files = ( 'proxy-server.conf', 'object-server/1.conf', 'object-server/2.conf', @@ -1007,7 +1007,7 @@ class TestServer(unittest.TestCase): class MockSpawn(): def __init__(self, pids=None): - self.ini_files = [] + self.conf_files = [] self.kwargs = [] if not pids: def one_forever(): @@ -1017,12 +1017,12 @@ class TestServer(unittest.TestCase): else: self.pids = (x for x in pids) - def __call__(self, ini_file, **kwargs): - self.ini_files.append(ini_file) + def __call__(self, conf_file, **kwargs): + self.conf_files.append(conf_file) self.kwargs.append(kwargs) return self.pids.next() - with temptree(ini_files) as swift_dir: + with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: @@ -1041,8 +1041,8 @@ class TestServer(unittest.TestCase): self.assertFalse(server.launch()) output = pop_stream(f) self.assert_('running' in output) - ini_file = self.join_swift_dir('proxy-server.conf') - self.assert_(ini_file in output) + conf_file = self.join_swift_dir('proxy-server.conf') + self.assert_(conf_file in output) pid_file = self.join_run_dir('proxy-server/2.pid') self.assert_(pid_file in output) self.assert_('already started' in output) @@ -1051,12 +1051,12 @@ class TestServer(unittest.TestCase): # test ignore once for non-start-once server mock_spawn = MockSpawn([1]) server.spawn = mock_spawn - ini_file = self.join_swift_dir('proxy-server.conf') + conf_file = self.join_swift_dir('proxy-server.conf') expected = { - 1: ini_file, + 1: conf_file, } self.assertEquals(server.launch(once=True), expected) - self.assertEquals(mock_spawn.ini_files, [ini_file]) + self.assertEquals(mock_spawn.conf_files, [conf_file]) expected = { 'once': False, } @@ -1079,7 +1079,7 @@ class TestServer(unittest.TestCase): 4: conf4, } self.assertEquals(server.launch(once=True), expected) - self.assertEquals(mock_spawn.ini_files, [conf1, conf2, + self.assertEquals(mock_spawn.conf_files, [conf1, conf2, conf3, conf4]) expected = { 'once': True, @@ -1094,7 +1094,7 @@ class TestServer(unittest.TestCase): 4: conf4, } self.assertEquals(server.launch(number=4), expected) - self.assertEquals(mock_spawn.ini_files, [conf4]) + self.assertEquals(mock_spawn.conf_files, [conf4]) expected = { 'number': 4 } @@ -1103,7 +1103,7 @@ class TestServer(unittest.TestCase): sys.stdout = old_stdout def test_stop(self): - ini_files = ( + conf_files = ( 'account-server/1.conf', 'account-server/2.conf', 'account-server/3.conf', @@ -1116,7 +1116,7 @@ class TestServer(unittest.TestCase): ('account-reaper/4.pid', 4), ) - with temptree(ini_files) as swift_dir: + with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: From 1f78fae2fcb8f8a4a7a2ea268e31f1545718a670 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Sat, 12 Feb 2011 14:50:24 -0600 Subject: [PATCH 51/64] more tests and cleanup --- bin/swift-init | 4 +- swift/common/manager.py | 16 ++-- test/unit/common/test_manager.py | 136 ++++++++++++++++++++++++++++--- 3 files changed, 137 insertions(+), 19 deletions(-) diff --git a/bin/swift-init b/bin/swift-init index c6e770bd8f..53418d64e1 100644 --- a/bin/swift-init +++ b/bin/swift-init @@ -19,7 +19,7 @@ from optparse import OptionParser from swift.common.manager import Server, Manager, UnknownCommandError -USAGE = """%prog [ +USAGE = """%prog [ ...] Commands: """ + '\n'.join(["%16s: %s" % x for x in Manager.list_commands()]) @@ -63,7 +63,7 @@ def main(): except UnknownCommandError: parser.print_help() print 'ERROR: unknown command, %s' % command - return 1 + status = 1 return 1 if status else 0 diff --git a/swift/common/manager.py b/swift/common/manager.py index d2a8c3f0d0..881cfa4457 100644 --- a/swift/common/manager.py +++ b/swift/common/manager.py @@ -28,6 +28,7 @@ import re SWIFT_DIR = '/etc/swift' RUN_DIR = '/var/run/swift' +# auth-server has been removed from ALL_SERVERS, start it explicitly ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor', 'container-replicator', 'container-server', 'container-updater', 'object-auditor', 'object-server', 'object-replicator', 'object-updater', @@ -35,7 +36,7 @@ ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor', MAIN_SERVERS = ['proxy-server', 'account-server', 'container-server', 'object-server'] REST_SERVERS = [s for s in ALL_SERVERS if s not in MAIN_SERVERS] -GRACEFUL_SHUTDOWN_SERVERS = MAIN_SERVERS +GRACEFUL_SHUTDOWN_SERVERS = MAIN_SERVERS + ['auth-server'] START_ONCE_SERVERS = REST_SERVERS KILL_WAIT = 15 # seconds to wait for servers to die @@ -210,7 +211,8 @@ class Manager(): server_pids[server] = signaled_pids # all signaled_pids, i.e. list(itertools.chain(*server_pids.values())) - signaled_pids = [p for server, pids in server_pids.items() for p in pids] + signaled_pids = [p for server, pids in server_pids.items() + for p in pids] # keep track of the pids yeiled back as killed for all servers killed_pids = set() for server, killed_pid in watch_server_pids(server_pids, @@ -253,9 +255,9 @@ class Manager(): kwargs['graceful'] = True status = 0 for server in self.servers: - init = Manager([server.server]) - status += init.stop(**kwargs) - status += init.start(**kwargs) + m = Manager([server.server]) + status += m.stop(**kwargs) + status += m.start(**kwargs) return status @command @@ -398,8 +400,8 @@ class Server(): number = kwargs.get('number', 0) if number: conf_files = self.conf_files(**kwargs) - # limt pid_files the one who translates to the indexed conf_file for - # this given number + # limt pid_files the one who translates to the indexed + # conf_file for this given number pid_files = [pid_file for pid_file in pid_files if self.get_conf_file_name(pid_file) in conf_files] return pid_files diff --git a/test/unit/common/test_manager.py b/test/unit/common/test_manager.py index de39a13cb7..f1def168ab 100644 --- a/test/unit/common/test_manager.py +++ b/test/unit/common/test_manager.py @@ -326,7 +326,7 @@ class TestServer(unittest.TestCase): self.assertEquals(conf_file, server.get_conf_file_name(pid_file)) def test_conf_files(self): - # test get single ini file + # test get single conf file conf_files = ( 'proxy-server.conf', 'proxy-server.ini', @@ -653,7 +653,7 @@ class TestServer(unittest.TestCase): self.assertEquals(manager.os.pid_sigs[1], [signal.SIGHUP]) # start up other servers manager.os = MockOs([11, 12]) - # test multi server kill & ignore graceful on unsupport server + # test multi server kill & ignore graceful on unsupported server self.assertFalse('object-replicator' in manager.GRACEFUL_SHUTDOWN_SERVERS) server = manager.Server('object-replicator') @@ -993,6 +993,7 @@ class TestServer(unittest.TestCase): # stubs conf_files = ( 'proxy-server.conf', + 'auth-server.conf', 'object-server/1.conf', 'object-server/2.conf', 'object-server/3.conf', @@ -1020,7 +1021,11 @@ class TestServer(unittest.TestCase): def __call__(self, conf_file, **kwargs): self.conf_files.append(conf_file) self.kwargs.append(kwargs) - return self.pids.next() + rv = self.pids.next() + if isinstance(rv, Exception): + raise rv + else: + return rv with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir @@ -1099,6 +1104,13 @@ class TestServer(unittest.TestCase): 'number': 4 } self.assertEquals(mock_spawn.kwargs, [expected]) + # test cmd does not exist + server = manager.Server('auth') + mock_spawn = MockSpawn([OSError(errno.ENOENT, 'blah')]) + server.spawn = mock_spawn + self.assertEquals(server.launch(), {}) + self.assert_('swift-auth-server does not exist' in + pop_stream(f)) finally: sys.stdout = old_stdout @@ -1455,7 +1467,6 @@ class TestManager(unittest.TestCase): def __call__(self, server): return MockServerFactory.MockServer(self.server_pids[server]) - def mock_watch_server_pids(server_pids, **kwargs): for server, pids in server_pids.items(): for pid in pids: @@ -1498,19 +1509,112 @@ class TestManager(unittest.TestCase): # TODO: more tests def test_shutdown(self): - pass + m = manager.Manager(['test']) + m.stop_was_called = False + + def mock_stop(*args, **kwargs): + m.stop_was_called = True + expected = {'graceful': True} + self.assertEquals(kwargs, expected) + return 0 + m.stop = mock_stop + status = m.shutdown() + self.assertEquals(status, 0) + self.assertEquals(m.stop_was_called, True) def test_restart(self): - pass + m = manager.Manager(['test']) + m.stop_was_called = False + + def mock_stop(*args, **kwargs): + m.stop_was_called = True + return 0 + m.start_was_called = False + + def mock_start(*args, **kwargs): + m.start_was_called = True + return 0 + m.stop = mock_stop + m.start = mock_start + status = m.restart() + self.assertEquals(status, 0) + self.assertEquals(m.stop_was_called, True) + self.assertEquals(m.start_was_called, True) def test_reload(self): - pass + class MockManager(): + called = defaultdict(list) + + def __init__(self, servers): + pass + + @classmethod + def reset_called(cls): + cls.called = defaultdict(list) + + def stop(self, **kwargs): + MockManager.called['stop'].append(kwargs) + return 0 + + def start(self, **kwargs): + MockManager.called['start'].append(kwargs) + return 0 + + _orig_manager = manager.Manager + try: + m = _orig_manager(['auth']) + for server in m.servers: + self.assert_(server.server in + manager.GRACEFUL_SHUTDOWN_SERVERS) + manager.Manager = MockManager + status = m.reload() + self.assertEquals(status, 0) + expected = { + 'start': [{'graceful': True}], + 'stop': [{'graceful': True}], + } + self.assertEquals(MockManager.called, expected) + # test force graceful + MockManager.reset_called() + m = _orig_manager(['*-server']) + self.assert_(len(m.servers), 4) + for server in m.servers: + self.assert_(server.server in + manager.GRACEFUL_SHUTDOWN_SERVERS) + manager.Manager = MockManager + status = m.reload(graceful=False) + self.assertEquals(status, 0) + expected = { + 'start': [{'graceful': True}] * 4, + 'stop': [{'graceful': True}] * 4, + } + self.assertEquals(MockManager.called, expected) + + finally: + manager.Manager = _orig_manager def test_force_reload(self): - pass + m = manager.Manager(['test']) + m.reload_was_called = False + + def mock_reload(*args, **kwargs): + m.reload_was_called = True + return 0 + m.reload = mock_reload + status = m.force_reload() + self.assertEquals(status, 0) + self.assertEquals(m.reload_was_called, True) def test_get_command(self): - pass + m = manager.Manager(['test']) + self.assertEquals(m.start, m.get_command('start')) + self.assertEquals(m.force_reload, m.get_command('force-reload')) + self.assertEquals(m.get_command('force-reload'), + m.get_command('force_reload')) + self.assertRaises(manager.UnknownCommandError, m.get_command, + 'no_command') + self.assertRaises(manager.UnknownCommandError, m.get_command, + '__init__') def test_list_commands(self): for cmd, help in manager.Manager.list_commands(): @@ -1520,8 +1624,20 @@ class TestManager(unittest.TestCase): self.assertEquals(method.__doc__.strip(), help) def test_run_command(self): - pass + m = manager.Manager(['test']) + m.cmd_was_called = False + def mock_cmd(*args, **kwargs): + m.cmd_was_called = True + expected = {'kw1': True, 'kw2': False} + self.assertEquals(kwargs, expected) + return 0 + mock_cmd.publicly_accessible = True + m.mock_cmd = mock_cmd + kwargs = {'kw1': True, 'kw2': False} + status = m.run_command('mock_cmd', **kwargs) + self.assertEquals(status, 0) + self.assertEquals(m.cmd_was_called, True) if __name__ == '__main__': unittest.main() From 694fa02b6741148b7e7a4c04da9be0af3c6f6708 Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Sat, 12 Feb 2011 18:27:59 -0800 Subject: [PATCH 52/64] Added missing lockfile configuration to sample rsync.conf --- etc/rsyncd.conf-sample | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/rsyncd.conf-sample b/etc/rsyncd.conf-sample index 2f0c9a84e2..c3b9952b16 100644 --- a/etc/rsyncd.conf-sample +++ b/etc/rsyncd.conf-sample @@ -7,13 +7,16 @@ pid file = /var/run/rsyncd.pid max connections = 2 path = /srv/node read only = false +lock file = /var/lock/account.lock [container] max connections = 4 path = /srv/node read only = false +lock file = /var/lock/container.lock [object] max connections = 8 path = /srv/node read only = false +lock file = /var/lock/object.lock From 22a45b3550a53a22ad7ae93d1cea7652efe7f868 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Mon, 14 Feb 2011 14:52:49 -0600 Subject: [PATCH 53/64] review cleanup --- doc/source/development_saio.rst | 1 - doc/source/misc.rst | 7 +++++++ swift/common/manager.py | 31 ++++++++++++++----------------- swift/common/utils.py | 14 +++++++------- swift/common/wsgi.py | 3 ++- test/unit/common/test_manager.py | 9 +++++++-- 6 files changed, 37 insertions(+), 28 deletions(-) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 976a743c51..38c0475975 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -531,7 +531,6 @@ Setting up scripts for running Swift #!/bin/bash swift-init all stop - sleep 5 sudo umount /mnt/sdb1 sudo mkfs.xfs -f -i size=1024 /dev/sdb1 sudo mount /mnt/sdb1 diff --git a/doc/source/misc.rst b/doc/source/misc.rst index 6d6ae04dbd..db77e464f3 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -116,6 +116,13 @@ MemCacheD :members: :show-inheritance: +Manager +========= + +.. automodule:: swift.common.manager + :members: + :show-inheritance: + Ratelimit ========= diff --git a/swift/common/manager.py b/swift/common/manager.py index 881cfa4457..6035be26ab 100644 --- a/swift/common/manager.py +++ b/swift/common/manager.py @@ -21,10 +21,11 @@ import resource import signal import sys import time -from swift.common.utils import search_tree, remove_file, write_file import subprocess import re +from swift.common.utils import search_tree, remove_file, write_file + SWIFT_DIR = '/etc/swift' RUN_DIR = '/var/run/swift' @@ -58,7 +59,6 @@ def setup_env(): " Running as non-root?" os.environ['PYTHON_EGG_CACHE'] = '/tmp' - return def command(func): @@ -88,7 +88,7 @@ def watch_server_pids(server_pids, interval=1, **kwargs): start = time.time() end = start + interval server_pids = dict(server_pids) # make a copy - while interval: + while True: for server, pids in server_pids.items(): for pid in pids: try: @@ -112,7 +112,6 @@ def watch_server_pids(server_pids, interval=1, **kwargs): break else: time.sleep(0.1) - return class UnknownCommandError(Exception): @@ -227,7 +226,8 @@ class Manager(): for server, pids in server_pids.items(): if not killed_pids.issuperset(pids): # some pids of this server were not killed - print 'Waited 15 seconds for %s to die; giving up' % (server) + print 'Waited %s seconds for %s to die; giving up' % ( + KILL_WAIT, server) return 1 @command @@ -314,7 +314,7 @@ class Server(): if '-' not in server: server = '%s-server' % server self.server = server.lower() - self.type = '-'.join(server.split('-')[:-1]) + self.type = server.rsplit('-', 1)[0] self.cmd = 'swift-%s' % server self.procs = [] @@ -360,11 +360,11 @@ class Server(): '.pid', 1)[0] + '.conf' def conf_files(self, **kwargs): - """Get ini files for this server + """Get conf files for this server :param: number, if supplied will only lookup the nth server - :returns: list of ini files + :returns: list of conf files """ found_conf_files = search_tree(SWIFT_DIR, '%s-server*' % self.type, '.conf') @@ -397,11 +397,9 @@ class Server(): :returns: list of pid files """ pid_files = search_tree(RUN_DIR, '%s*' % self.server, '.pid') - number = kwargs.get('number', 0) - if number: + if kwargs.get('number', 0): conf_files = self.conf_files(**kwargs) - # limt pid_files the one who translates to the indexed - # conf_file for this given number + # filter pid_files to match the index of numbered conf_file pid_files = [pid_file for pid_file in pid_files if self.get_conf_file_name(pid_file) in conf_files] return pid_files @@ -429,7 +427,7 @@ class Server(): os.kill(pid, sig) except OSError, e: #print '%s sig err: %s' % (pid, e) - if e.errno == 3: + if e.errno == errno.ESRCH: # pid does not exist if kwargs.get('verbose'): print "Removing stale pid file %s" % pid_file @@ -533,6 +531,7 @@ class Server(): output = proc.stdout.read() if output: print output + proc.communicate() if proc.returncode: status += 1 return status @@ -551,7 +550,7 @@ class Server(): def launch(self, **kwargs): """ - Collect ini files and attempt to spawn the processes for this server + Collect conf files and attempt to spawn the processes for this server """ conf_files = self.conf_files(**kwargs) if not conf_files: @@ -581,8 +580,6 @@ class Server(): if self.server not in START_ONCE_SERVERS: kwargs['once'] = False - # TODO: check if self.cmd exists? - pids = {} for conf_file in conf_files: if kwargs.get('once'): @@ -594,7 +591,7 @@ class Server(): pid = self.spawn(conf_file, **kwargs) except OSError, e: if e.errno == errno.ENOENT: - # cmd does not exist + # TODO: should I check if self.cmd exists earlier? print "%s does not exist" % self.cmd break pids[pid] = conf_file diff --git a/swift/common/utils.py b/swift/common/utils.py index e185e864b5..744f65c997 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -77,9 +77,8 @@ TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes', 'on', 'On')) def validate_configuration(): if HASH_PATH_SUFFIX == '': - print "Error: [swift-hash]: swift_hash_path_suffix missing from " \ - "/etc/swift/swift.conf" - sys.exit(1) + sys.exit("Error: [swift-hash]: swift_hash_path_suffix missing " + "from /etc/swift/swift.conf") def load_libc_function(func_name): @@ -822,13 +821,14 @@ def write_file(path, contents): :param contents: data to write to file, will be converted to string """ - dir, name = os.path.split(path) - if not os.path.exists(dir): + dirname, name = os.path.split(path) + if not os.path.exists(dirname): try: - os.makedirs(dir) + os.makedirs(dirname) except OSError, err: if err.errno == errno.EACCES: - sys.exit('Unable to create %s. Running as non-root?' % dir) + sys.exit('Unable to create %s. Running as ' + 'non-root?' % dirname) with open(path, 'w') as f: f.write('%s' % contents) diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 4e4c26650a..4bbf47d75a 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -119,9 +119,10 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): logger = get_logger(conf, log_name, log_to_console=kwargs.pop('verbose', False), log_route='wsgi') + # TODO: should we wait to close stdio until after we've created the socket + # and initialized the app? # redirect errors to logger and close stdio capture_stdio(logger) - # bind to address and port sock = get_socket(conf, default_port=kwargs.get('default_port', 8080)) # remaining tasks should not require elevated privileges diff --git a/test/unit/common/test_manager.py b/test/unit/common/test_manager.py index f1def168ab..6e9e2d3a85 100644 --- a/test/unit/common/test_manager.py +++ b/test/unit/common/test_manager.py @@ -204,7 +204,7 @@ class TestManagerModule(unittest.TestCase): # basic test, server dies gen = manager.watch_server_pids(server_pids) expected = [(server, 1)] - self.assertEquals([x for x in gen], [(server, 1)]) + self.assertEquals([x for x in gen], expected) # start long running server and short interval server = MockServer([1], zombie=15) server_pids = { @@ -886,7 +886,10 @@ class TestServer(unittest.TestCase): self.finished = False self.returncode = None if fail_to_start: + self._returncode = 1 self.run = self.fail + else: + self._returncode = 0 def __enter__(self): self.start() @@ -908,9 +911,11 @@ class TestServer(unittest.TestCase): print >>self._stdout, 'mock process started' sleep(self.delay) # perform setup processing print >>self._stdout, 'mock process failed to start' - self.returncode = 1 self.close_stdout() + def communicate(self): + self.returncode = self._returncode + def run(self): print >>self._stdout, 'mock process started' sleep(self.delay) # perform setup processing From da794932ba5e627c2aaa58200116eeaf5234863b Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Mon, 14 Feb 2011 16:53:25 -0600 Subject: [PATCH 54/64] i18n --- swift/common/manager.py | 48 ++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/swift/common/manager.py b/swift/common/manager.py index 6035be26ab..4a7e782d72 100644 --- a/swift/common/manager.py +++ b/swift/common/manager.py @@ -55,8 +55,8 @@ def setup_env(): resource.setrlimit(resource.RLIMIT_DATA, (MAX_MEMORY, MAX_MEMORY)) except ValueError: - print "WARNING: Unable to increase file descriptor limit." \ - " Running as non-root?" + print _("WARNING: Unable to increase file descriptor limit. " + "Running as non-root?") os.environ['PYTHON_EGG_CACHE'] = '/tmp' @@ -168,7 +168,7 @@ class Manager(): try: status += server.interact(**kwargs) except KeyboardInterrupt: - print '\nuser quit' + print _('\nuser quit') self.stop(**kwargs) break elif kwargs.get('wait', False): @@ -205,7 +205,7 @@ class Manager(): for server in self.servers: signaled_pids = server.stop(**kwargs) if not signaled_pids: - print 'No %s running' % server + print _('No %s running') % server else: server_pids[server] = signaled_pids @@ -216,7 +216,7 @@ class Manager(): killed_pids = set() for server, killed_pid in watch_server_pids(server_pids, interval=KILL_WAIT, **kwargs): - print "%s (%s) appears to have stopped" % (server, killed_pid) + print _("%s (%s) appears to have stopped") % (server, killed_pid) killed_pids.add(killed_pid) if not killed_pids.symmetric_difference(signaled_pids): # all proccesses have been stopped @@ -226,7 +226,7 @@ class Manager(): for server, pids in server_pids.items(): if not killed_pids.issuperset(pids): # some pids of this server were not killed - print 'Waited %s seconds for %s to die; giving up' % ( + print _('Waited %s seconds for %s to die; giving up') % ( KILL_WAIT, server) return 1 @@ -379,13 +379,13 @@ class Server(): if not conf_files: # maybe there's a config file(s) out there, but I couldn't find it! if not kwargs.get('quiet'): - print('Unable to locate config %sfor %s' % ( - ('number %s ' % number if number else ''), self.server)) + print _('Unable to locate config %sfor %s') % ( + ('number %s ' % number if number else ''), self.server) if kwargs.get('verbose') and not kwargs.get('quiet'): if found_conf_files: - print('Found configs:') + print _('Found configs:') for i, conf_file in enumerate(found_conf_files): - print(' %d) %s' % (i + 1, conf_file)) + print ' %d) %s' % (i + 1, conf_file) return conf_files @@ -422,15 +422,14 @@ class Server(): for pid_file, pid in self.iter_pid_files(**kwargs): try: if sig != signal.SIG_DFL: - print 'Signal %s pid: %s signal: %s' % ( - self.server, pid, sig) + print _('Signal %s pid: %s signal: %s') % (self.server, + pid, sig) os.kill(pid, sig) except OSError, e: - #print '%s sig err: %s' % (pid, e) if e.errno == errno.ESRCH: # pid does not exist if kwargs.get('verbose'): - print "Removing stale pid file %s" % pid_file + print _("Removing stale pid file %s") % pid_file remove_file(pid_file) else: # process exists @@ -476,14 +475,14 @@ class Server(): kwargs['quiet'] = True conf_files = self.conf_files(**kwargs) if conf_files: - print "%s #%d not running (%s)" % (self.server, number, - conf_files[0]) + print _("%s #%d not running (%s)") % (self.server, number, + conf_files[0]) else: - print "No %s running" % self.server + print _("No %s running") % self.server return 1 for pid, pid_file in pids.items(): conf_file = self.get_conf_file_name(pid_file) - print "%s running (%s - %s)" % (self.server, pid, conf_file) + print _("%s running (%s - %s)") % (self.server, pid, conf_file) return 0 def spawn(self, conf_file, once=False, wait=False, daemon=True, **kwargs): @@ -567,14 +566,13 @@ class Server(): # any unstarted instances if conf_file in conf_files: already_started = True - print "%s running (%s - %s)" % (self.server, pid, conf_file) + print _("%s running (%s - %s)") % (self.server, pid, conf_file) elif not kwargs.get('number', 0): already_started = True - print "%s running (%s - %s)" % (self.server, pid, pid_file) + print _("%s running (%s - %s)") % (self.server, pid, pid_file) if already_started: - print "%s already started..." % self.server - #self.status(pids) + print _("%s already started...") % self.server return [] if self.server not in START_ONCE_SERVERS: @@ -583,16 +581,16 @@ class Server(): pids = {} for conf_file in conf_files: if kwargs.get('once'): - msg = 'Running %s once' % self.server + msg = _('Running %s once') % self.server else: - msg = 'Starting %s' % self.server + msg = _('Starting %s') % self.server print '%s...(%s)' % (msg, conf_file) try: pid = self.spawn(conf_file, **kwargs) except OSError, e: if e.errno == errno.ENOENT: # TODO: should I check if self.cmd exists earlier? - print "%s does not exist" % self.cmd + print _("%s does not exist") % self.cmd break pids[pid] = conf_file From c1884bbfdda145b64636a461ceba1b74591593b7 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Mon, 14 Feb 2011 17:02:08 -0600 Subject: [PATCH 55/64] redbo says we should capture stdio later --- swift/common/daemon.py | 2 +- swift/common/wsgi.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/swift/common/daemon.py b/swift/common/daemon.py index 91230e4d2b..9f4f004508 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -39,8 +39,8 @@ class Daemon(object): def run(self, once=False, **kwargs): """Run the daemon""" utils.validate_configuration() - utils.capture_stdio(self.logger, **kwargs) utils.drop_privileges(self.conf.get('user', 'swift')) + utils.capture_stdio(self.logger, **kwargs) def kill_children(*args): signal.signal(signal.SIGTERM, signal.SIG_IGN) diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 4bbf47d75a..5f4494b736 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -119,10 +119,6 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): logger = get_logger(conf, log_name, log_to_console=kwargs.pop('verbose', False), log_route='wsgi') - # TODO: should we wait to close stdio until after we've created the socket - # and initialized the app? - # redirect errors to logger and close stdio - capture_stdio(logger) # bind to address and port sock = get_socket(conf, default_port=kwargs.get('default_port', 8080)) # remaining tasks should not require elevated privileges @@ -131,6 +127,9 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # finally after binding to ports and privilege drop, run app __init__ code app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name}) + # redirect errors to logger and close stdio + capture_stdio(logger) + def run_server(): wsgi.HttpProtocol.default_request_version = "HTTP/1.0" eventlet.hubs.use_hub('poll') From 6766bd371acff422cbe2685cd364d638792e6610 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 15 Feb 2011 10:48:22 -0600 Subject: [PATCH 56/64] wait is on by default --- bin/swift-init | 5 +++-- swift/common/manager.py | 14 +++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/bin/swift-init b/bin/swift-init index 53418d64e1..d0eee0daba 100644 --- a/bin/swift-init +++ b/bin/swift-init @@ -29,8 +29,9 @@ def main(): parser = OptionParser(USAGE) parser.add_option('-v', '--verbose', action="store_true", default=False, help="display verbose output") - parser.add_option('-w', '--wait', action="store_true", default=False, - help="wait for server to start before returning") + parser.add_option('-w', '--no-wait', action="store_false", dest="wait", + default=True, help="won't wait for server to start " + "before returning") parser.add_option('-o', '--once', action="store_true", default=False, help="only run one pass of daemon") # this is a negative option, default is options.daemon = True diff --git a/swift/common/manager.py b/swift/common/manager.py index 4a7e782d72..b5b126a822 100644 --- a/swift/common/manager.py +++ b/swift/common/manager.py @@ -171,21 +171,21 @@ class Manager(): print _('\nuser quit') self.stop(**kwargs) break - elif kwargs.get('wait', False): + elif kwargs.get('wait', True): for server in self.servers: status += server.wait(**kwargs) return status @command - def wait(self, **kwargs): - """spawn server and wait for it to start + def no_wait(self, **kwargs): + """spawn server and return immediately """ - kwargs['wait'] = True + kwargs['wait'] = False return self.start(**kwargs) @command def no_daemon(self, **kwargs): - """start a server interactivly + """start a server interactively """ kwargs['daemon'] = False return self.start(**kwargs) @@ -485,7 +485,7 @@ class Server(): print _("%s running (%s - %s)") % (self.server, pid, conf_file) return 0 - def spawn(self, conf_file, once=False, wait=False, daemon=True, **kwargs): + def spawn(self, conf_file, once=False, wait=True, daemon=True, **kwargs): """Launch a subprocess for this server. :param conf_file: path to conf_file to use as first arg @@ -542,7 +542,7 @@ class Server(): status = 0 for proc in self.procs: # wait for process to terminate - proc.communicate()[0] + proc.communicate() if proc.returncode: status += 1 return status From a8b4f859c0c1b13011c3b0957d70351be393e424 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 15 Feb 2011 13:19:33 -0600 Subject: [PATCH 57/64] fixed tests for new wait default true --- test/unit/common/test_manager.py | 98 ++++++++++++++------------------ 1 file changed, 43 insertions(+), 55 deletions(-) diff --git a/test/unit/common/test_manager.py b/test/unit/common/test_manager.py index 6e9e2d3a85..47a50c6a24 100644 --- a/test/unit/common/test_manager.py +++ b/test/unit/common/test_manager.py @@ -807,10 +807,8 @@ class TestServer(unittest.TestCase): conf_file, ] self.assertEquals(proc.args, expected_args) - # assert stdout is /dev/null - self.assert_(isinstance(proc.stdout, file)) - self.assertEquals(proc.stdout.name, os.devnull) - self.assertEquals(proc.stdout.mode, 'w+b') + # assert stdout is piped + self.assertEquals(proc.stdout, MockProcess.PIPE) self.assertEquals(proc.stderr, proc.stdout) # test multi server process calls spawn multiple times manager.subprocess = MockProcess([11, 12, 13, 14]) @@ -825,11 +823,8 @@ class TestServer(unittest.TestCase): self.assertEquals(len(server.procs), 1) proc = server.procs[0] expected_args = ['swift-test-server', conf1, 'once'] - self.assertEquals(proc.args, expected_args) - # assert stdout is /dev/null - self.assert_(isinstance(proc.stdout, file)) - self.assertEquals(proc.stdout.name, os.devnull) - self.assertEquals(proc.stdout.mode, 'w+b') + # assert stdout is piped + self.assertEquals(proc.stdout, MockProcess.PIPE) self.assertEquals(proc.stderr, proc.stdout) # test server not daemon server.spawn(conf2, daemon=False) @@ -842,15 +837,17 @@ class TestServer(unittest.TestCase): self.assertEquals(proc.stdout, None) self.assertEquals(proc.stderr, None) # test server wait - server.spawn(conf3, wait=True) + server.spawn(conf3, wait=False) self.assert_(server.procs) self.assertEquals(len(server.procs), 3) proc = server.procs[2] - # assert stdout is piped - self.assertEquals(proc.stdout, MockProcess.PIPE) + # assert stdout is /dev/null + self.assert_(isinstance(proc.stdout, file)) + self.assertEquals(proc.stdout.name, os.devnull) + self.assertEquals(proc.stdout.mode, 'w+b') self.assertEquals(proc.stderr, proc.stdout) # test not daemon over-rides wait - server.spawn(conf4, wait=True, daemon=False, once=True) + server.spawn(conf4, wait=False, daemon=False, once=True) self.assert_(server.procs) self.assertEquals(len(server.procs), 4) proc = server.procs[3] @@ -1275,10 +1272,7 @@ class TestManager(unittest.TestCase): def wait(self, **kwargs): self.called['wait'].append(kwargs) - if 'error' in self.server: - return 1 - else: - return 0 + return int('error' in self.server) def stop(self, **kwargs): self.called['stop'].append(kwargs) @@ -1299,20 +1293,19 @@ class TestManager(unittest.TestCase): manager.Server = MockServer # test no errors on launch - m = manager.Manager(['proxy', 'error']) + m = manager.Manager(['proxy']) status = m.start() self.assertEquals(status, 0) for server in m.servers: self.assertEquals(server.called['launch'], [{}]) - # test error on wait + # test error on launch m = manager.Manager(['proxy', 'error']) - kwargs = {'wait': True} - status = m.start(**kwargs) + status = m.start() self.assertEquals(status, 1) for server in m.servers: - self.assertEquals(server.called['launch'], [kwargs]) - self.assertEquals(server.called['wait'], [kwargs]) + self.assertEquals(server.called['launch'], [{}]) + self.assertEquals(server.called['wait'], [{}]) # test interact m = manager.Manager(['proxy', 'error']) @@ -1330,7 +1323,7 @@ class TestManager(unittest.TestCase): manager.setup_env = old_setup_env manager.Server = old_swift_server - def test_wait(self): + def test_no_wait(self): class MockServer(): def __init__(self, server): self.server = server @@ -1348,47 +1341,35 @@ class TestManager(unittest.TestCase): manager.Server = MockServer # test success init = manager.Manager(['proxy']) - status = init.wait() + status = init.no_wait() + self.assertEquals(status, 0) + for server in init.servers: + self.assertEquals(len(server.called['launch']), 1) + called_kwargs = server.called['launch'][0] + self.assertFalse(called_kwargs['wait']) + self.assertFalse(server.called['wait']) + # test no errocode status even on error + init = manager.Manager(['error']) + status = init.no_wait() self.assertEquals(status, 0) for server in init.servers: self.assertEquals(len(server.called['launch']), 1) called_kwargs = server.called['launch'][0] self.assert_('wait' in called_kwargs) - self.assert_(called_kwargs['wait']) - self.assertEquals(len(server.called['wait']), 1) - called_kwargs = server.called['wait'][0] - self.assert_('wait' in called_kwargs) - self.assert_(called_kwargs['wait']) - # test error - init = manager.Manager(['error']) - status = init.wait() - self.assertEquals(status, 1) - for server in init.servers: - self.assertEquals(len(server.called['launch']), 1) - called_kwargs = server.called['launch'][0] - self.assert_('wait' in called_kwargs) - self.assert_(called_kwargs['wait']) - self.assertEquals(len(server.called['wait']), 1) - called_kwargs = server.called['wait'][0] - self.assert_('wait' in called_kwargs) - self.assert_(called_kwargs['wait']) + self.assertFalse(called_kwargs['wait']) + self.assertFalse(server.called['wait']) # test wait with once option init = manager.Manager(['updater', 'replicator-error']) - status = init.wait(once=True) - self.assertEquals(status, 1) + status = init.no_wait(once=True) + self.assertEquals(status, 0) for server in init.servers: self.assertEquals(len(server.called['launch']), 1) called_kwargs = server.called['launch'][0] self.assert_('wait' in called_kwargs) - self.assert_(called_kwargs['wait']) - self.assert_('once' in called_kwargs) - self.assert_(called_kwargs['once']) - self.assertEquals(len(server.called['wait']), 1) - called_kwargs = server.called['wait'][0] - self.assert_('wait' in called_kwargs) - self.assert_(called_kwargs['wait']) + self.assertFalse(called_kwargs['wait']) self.assert_('once' in called_kwargs) self.assert_(called_kwargs['once']) + self.assertFalse(server.called['wait']) finally: manager.Server = orig_swift_server @@ -1434,6 +1415,13 @@ class TestManager(unittest.TestCase): self.server = server self.called = defaultdict(list) + def wait(self, **kwargs): + self.called['wait'].append(kwargs) + if 'error' in self.server: + return 1 + else: + return 0 + def launch(self, **kwargs): return self.called['launch'].append(kwargs) @@ -1444,15 +1432,15 @@ class TestManager(unittest.TestCase): init = manager.Manager(['account-reaper']) status = init.once() self.assertEquals(status, 0) - # test no error code on error + # test error code on error init = manager.Manager(['error-reaper']) status = init.once() - self.assertEquals(status, 0) + self.assertEquals(status, 1) for server in init.servers: self.assertEquals(len(server.called['launch']), 1) called_kwargs = server.called['launch'][0] self.assertEquals(called_kwargs, {'once': True}) - self.assertEquals(len(server.called['wait']), 0) + self.assertEquals(len(server.called['wait']), 1) self.assertEquals(len(server.called['interact']), 0) finally: manager.Server = orig_swift_server From fc6391ea5c623c4c94ef37cf7c322b621b24313d Mon Sep 17 00:00:00 2001 From: gholt Date: Tue, 15 Feb 2011 18:43:55 -0800 Subject: [PATCH 58/64] ring: pickles now use only stdlib objects; old and really old pickles can still be read --- bin/swift-ring-builder | 43 ++++++++++++++++++++++-------------- swift/common/ring/builder.py | 43 ++++++++++++++++++++++++++++++++++++ swift/common/ring/ring.py | 8 +++++++ 3 files changed, 78 insertions(+), 16 deletions(-) diff --git a/bin/swift-ring-builder b/bin/swift-ring-builder index c6d91f92b4..fd24a1d93f 100755 --- a/bin/swift-ring-builder +++ b/bin/swift-ring-builder @@ -19,7 +19,7 @@ from errno import EEXIST from gzip import GzipFile from os import mkdir from os.path import basename, dirname, exists, join as pathjoin -from sys import argv, exit +from sys import argv, exit, modules from textwrap import wrap from time import time @@ -153,9 +153,9 @@ swift-ring-builder create except OSError, err: if err.errno != EEXIST: raise - pickle.dump(builder, open(pathjoin(backup_dir, + pickle.dump(builder.to_dict(), open(pathjoin(backup_dir, '%d.' % time() + basename(argv[1])), 'wb'), protocol=2) - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_CHANGED) def default(): @@ -312,7 +312,7 @@ swift-ring-builder add z-:/_ else: print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \ (zone, ip, port, device_name, meta, weight, next_dev_id) - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) def set_weight(): @@ -345,7 +345,7 @@ swift-ring-builder set_weight builder.set_dev_weight(dev['id'], weight) print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \ 'weight set to %(weight)s' % dev - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) def set_info(): @@ -427,7 +427,7 @@ swift-ring-builder set_info for key, value in change: dev[key] = value print 'Device %s is now %s' % (orig_dev_string, format_device(dev)) - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) def remove(): @@ -463,7 +463,7 @@ swift-ring-builder remove print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \ 'marked for removal and will be removed next rebalance.' \ % dev - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) def rebalance(): @@ -495,13 +495,14 @@ swift-ring-builder rebalance % builder.min_part_hours print '-' * 79 ts = time() - pickle.dump(builder.get_ring(), + pickle.dump(builder.get_ring().to_dict(), GzipFile(pathjoin(backup_dir, '%d.' % ts + basename(ring_file)), 'wb'), protocol=2) - pickle.dump(builder, open(pathjoin(backup_dir, + pickle.dump(builder.to_dict(), open(pathjoin(backup_dir, '%d.' % ts + basename(argv[1])), 'wb'), protocol=2) - pickle.dump(builder.get_ring(), GzipFile(ring_file, 'wb'), protocol=2) - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.get_ring().to_dict(), GzipFile(ring_file, 'wb'), + protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_CHANGED) def validate(): @@ -528,15 +529,15 @@ swift-ring-builder write_ring '"rebalance"?' else: print 'Warning: Writing an empty ring' - pickle.dump(ring_data, + pickle.dump(ring_data.to_dict(), GzipFile(pathjoin(backup_dir, '%d.' % time() + basename(ring_file)), 'wb'), protocol=2) - pickle.dump(ring_data, GzipFile(ring_file, 'wb'), protocol=2) + pickle.dump(ring_data.to_dict(), GzipFile(ring_file, 'wb'), protocol=2) exit(EXIT_RING_CHANGED) def pretend_min_part_hours_passed(): builder.pretend_min_part_hours_passed() - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) def set_min_part_hours(): @@ -552,7 +553,7 @@ swift-ring-builder set_min_part_hours builder.change_min_part_hours(int(argv[3])) print 'The minimum number of hours before a partition can be ' \ 'reassigned is now set to %s' % argv[3] - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) @@ -578,7 +579,17 @@ if __name__ == '__main__': exit(EXIT_RING_UNCHANGED) if exists(argv[1]): - builder = pickle.load(open(argv[1], 'rb')) + try: + builder = pickle.load(open(argv[1], 'rb')) + if not hasattr(builder, 'devs'): + builder_dict = builder + builder = RingBuilder(1, 1, 1) + builder.copy_from(builder_dict) + except ImportError: # Happens with really old builder pickles + modules['swift.ring_builder'] = \ + modules['swift.common.ring.builder'] + builder = RingBuilder(1, 1, 1) + builder.copy_from(pickle.load(open(argv[1], 'rb'))) for dev in builder.devs: if dev and 'meta' not in dev: dev['meta'] = '' diff --git a/swift/common/ring/builder.py b/swift/common/ring/builder.py index 3f728e307a..86e6cce287 100644 --- a/swift/common/ring/builder.py +++ b/swift/common/ring/builder.py @@ -69,6 +69,49 @@ class RingBuilder(object): self._remove_devs = [] self._ring = None + def copy_from(self, builder): + if hasattr(builder, 'devs'): + self.part_power = builder.part_power + self.replicas = builder.replicas + self.min_part_hours = builder.min_part_hours + self.parts = builder.parts + self.devs = builder.devs + self.devs_changed = builder.devs_changed + self.version = builder.version + self._replica2part2dev = builder._replica2part2dev + self._last_part_moves_epoch = builder._last_part_moves_epoch + self._last_part_moves = builder._last_part_moves + self._last_part_gather_start = builder._last_part_gather_start + self._remove_devs = builder._remove_devs + else: + self.part_power = builder['part_power'] + self.replicas = builder['replicas'] + self.min_part_hours = builder['min_part_hours'] + self.parts = builder['parts'] + self.devs = builder['devs'] + self.devs_changed = builder['devs_changed'] + self.version = builder['version'] + self._replica2part2dev = builder['_replica2part2dev'] + self._last_part_moves_epoch = builder['_last_part_moves_epoch'] + self._last_part_moves = builder['_last_part_moves'] + self._last_part_gather_start = builder['_last_part_gather_start'] + self._remove_devs = builder['_remove_devs'] + self._ring = None + + def to_dict(self): + return {'part_power': self.part_power, + 'replicas': self.replicas, + 'min_part_hours': self.min_part_hours, + 'parts': self.parts, + 'devs': self.devs, + 'devs_changed': self.devs_changed, + 'version': self.version, + '_replica2part2dev': self._replica2part2dev, + '_last_part_moves_epoch': self._last_part_moves_epoch, + '_last_part_moves': self._last_part_moves, + '_last_part_gather_start': self._last_part_gather_start, + '_remove_devs': self._remove_devs} + def change_min_part_hours(self, min_part_hours): """ Changes the value used to decide if a given partition can be moved diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 45ab407563..122a990fc5 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -29,6 +29,11 @@ class RingData(object): self._replica2part2dev_id = replica2part2dev_id self._part_shift = part_shift + def to_dict(self): + return {'devs': self.devs, + 'replica2part2dev_id': self._replica2part2dev_id, + 'part_shift': self._part_shift} + class Ring(object): """ @@ -47,6 +52,9 @@ class Ring(object): self._rtime = time() + self.reload_time if force or self.has_changed(): ring_data = pickle.load(GzipFile(self.pickle_gz_path, 'rb')) + if not hasattr(ring_data, 'devs'): + ring_data = RingData(ring_data['replica2part2dev_id'], + ring_data['devs'], ring_data['part_shift']) self._mtime = getmtime(self.pickle_gz_path) self.devs = ring_data.devs self.zone2devs = {} From ae1c2d73ab0b90f26ac85b2823c0ccd6e19f169b Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 16 Feb 2011 09:02:38 -0600 Subject: [PATCH 59/64] creating a Ring will ensure a valid HASH_PATH_SUFFIX To make sure that node lookups match what the servers return the generated hashes need to match. All the utils that use the ring should validate their HASH_PATH_SUFFIX. --- swift/common/ring/ring.py | 4 +++- test/unit/common/ring/test_ring.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 45ab407563..f7edd1cee2 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -18,7 +18,7 @@ from gzip import GzipFile from os.path import getmtime from struct import unpack_from from time import time -from swift.common.utils import hash_path +from swift.common.utils import hash_path, validate_configuration class RingData(object): @@ -39,6 +39,8 @@ class Ring(object): """ def __init__(self, pickle_gz_path, reload_time=15): + # can't use the ring unless HASH_PATH_SUFFIX is set + validate_configuration() self.pickle_gz_path = pickle_gz_path self.reload_time = reload_time self._reload(force=True) diff --git a/test/unit/common/ring/test_ring.py b/test/unit/common/ring/test_ring.py index ad72a4c990..51a2cf2ba8 100644 --- a/test/unit/common/ring/test_ring.py +++ b/test/unit/common/ring/test_ring.py @@ -69,6 +69,13 @@ class TestRing(unittest.TestCase): self.assertEquals(self.ring.devs, self.intended_devs) self.assertEquals(self.ring.reload_time, self.intended_reload_time) self.assertEquals(self.ring.pickle_gz_path, self.testgz) + # test invalid endcap + _orig_hash_path_suffix = utils.HASH_PATH_SUFFIX + try: + utils.HASH_PATH_SUFFIX = '' + self.assertRaises(SystemExit, ring.Ring, self.testgz) + finally: + utils.HASH_PATH_SUFFIX = _orig_hash_path_suffix def test_has_changed(self): self.assertEquals(self.ring.has_changed(), False) From 092608b1c928ccb6a2767c5eb33c95c956bbd1ad Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 16 Feb 2011 14:47:31 -0600 Subject: [PATCH 60/64] added [options] to command usage --- bin/swift-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/swift-init b/bin/swift-init index 53418d64e1..11d0656824 100644 --- a/bin/swift-init +++ b/bin/swift-init @@ -19,7 +19,7 @@ from optparse import OptionParser from swift.common.manager import Server, Manager, UnknownCommandError -USAGE = """%prog [ ...] +USAGE = """%prog [ ...] [options] Commands: """ + '\n'.join(["%16s: %s" % x for x in Manager.list_commands()]) From ddc2a4c6b293a4b6c233492150cd44f754d48919 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 16 Feb 2011 19:30:18 -0600 Subject: [PATCH 61/64] fixed glob pattern matching index offset error in stats' LogUploader --- swift/stats/log_uploader.py | 2 +- test/unit/stats/test_log_uploader.py | 137 ++++++++++++++++++++++++++- 2 files changed, 135 insertions(+), 4 deletions(-) diff --git a/swift/stats/log_uploader.py b/swift/stats/log_uploader.py index b425738938..e75a05c07d 100644 --- a/swift/stats/log_uploader.py +++ b/swift/stats/log_uploader.py @@ -78,7 +78,7 @@ class LogUploader(Daemon): i = [(self.filename_format.index(c), c) for c in '%Y %m %d %H'.split()] i.sort() year_offset = month_offset = day_offset = hour_offset = None - base_offset = len(self.log_dir) + base_offset = len(self.log_dir.rstrip('/')) + 1 for start, c in i: offset = base_offset + start if c == '%Y': diff --git a/test/unit/stats/test_log_uploader.py b/test/unit/stats/test_log_uploader.py index 3585111750..145da9d710 100644 --- a/test/unit/stats/test_log_uploader.py +++ b/test/unit/stats/test_log_uploader.py @@ -13,16 +13,147 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: Tests +# TODO: More tests import unittest +import os +from datetime import datetime +from tempfile import mkdtemp +from shutil import rmtree + from swift.stats import log_uploader +import logging +logging.basicConfig(level=logging.DEBUG) +LOGGER = logging.getLogger() + +DEFAULT_GLOB = '%Y%m%d%H' class TestLogUploader(unittest.TestCase): - def test_placeholder(self): - pass + def test_upload_all_logs(self): + + class MockInternalProxy(): + + def create_container(self, *args, **kwargs): + pass + + class MonkeyLogUploader(log_uploader.LogUploader): + + def __init__(self, conf, logger=LOGGER): + self.log_dir = conf['log_dir'] + self.filename_format = conf.get('filename_format', DEFAULT_GLOB) + self.new_log_cutoff = 0 + self.logger = logger + self.internal_proxy = MockInternalProxy() + self.swift_account = '' + self.container_name = '' + + self.uploaded_files = [] + + def upload_one_log(self, filename, year, month, day, hour): + d = {'year': year, 'month': month, 'day': day, 'hour': hour} + self.uploaded_files.append((filename, d)) + + tmpdir = mkdtemp() + try: + today = datetime.now() + year = today.year + month = today.month + day = today.day + + today_str = today.strftime('%Y%m%d') + time_strs = [] + for i in range(24): + time_strs.append('%s%0.2d' % (today_str, i)) + for ts in time_strs: + open(os.path.join(tmpdir, ts), 'w').close() + + conf = {'log_dir': tmpdir} + uploader = MonkeyLogUploader(conf) + uploader.upload_all_logs() + self.assertEquals(len(uploader.uploaded_files), 24) + for i, file_date in enumerate(sorted(uploader.uploaded_files)): + d = {'year': year, 'month': month, 'day': day, 'hour': i} + for k, v in d.items(): + d[k] = '%0.2d' % v + expected = (os.path.join(tmpdir, '%s%0.2d' % (today_str, i)), d) + self.assertEquals(file_date, expected) + finally: + rmtree(tmpdir) + + tmpdir = mkdtemp() + try: + today = datetime.now() + year = today.year + month = today.month + day = today.day + + today_str = today.strftime('%Y%m%d') + time_strs = [] + for i in range(24): + time_strs.append('%s-%0.2d00' % (today_str, i)) + for ts in time_strs: + open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.gz' % ts), 'w').close() + + open(os.path.join(tmpdir, 'swift.blah_98764.%s-2400.tar.gz' % ts), 'w').close() + open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.g' % ts), 'w').close() + open(os.path.join(tmpdir, 'swift-blah_201102160100.%s-2400.tar.gz' % + '201102160100'), 'w').close() + + conf = { + 'log_dir': '%s/' % tmpdir, + 'filename_format': 'swift-blah_98764.%Y%m%d-%H*.tar.gz', + } + uploader = MonkeyLogUploader(conf) + uploader.upload_all_logs() + self.assertEquals(len(uploader.uploaded_files), 24) + for i, file_date in enumerate(sorted(uploader.uploaded_files)): + filename, date_dict = file_date + filename = os.path.basename(filename) + self.assert_(today_str in filename, filename) + self.assert_(filename.startswith('swift'), filename) + self.assert_(filename.endswith('tar.gz'), filename) + d = {'year': year, 'month': month, 'day': day, 'hour': i} + for k, v in d.items(): + d[k] = '%0.2d' % v + self.assertEquals(d, date_dict) + finally: + rmtree(tmpdir) + + tmpdir = mkdtemp() + try: + today = datetime.now() + year = today.year + month = today.month + day = today.day + + today_str = today.strftime('%Y%m%d') + time_strs = [] + for i in range(24): + time_strs.append('%s%0.2d' % (today_str, i)) + for i, ts in enumerate(time_strs): + open(os.path.join(tmpdir, '%s.%s.log' % (i, ts)), 'w').close() + + conf = { + 'log_dir': tmpdir, + 'filename_format': '*.%Y%m%d%H.log', + } + uploader = MonkeyLogUploader(conf) + uploader.upload_all_logs() + self.assertEquals(len(uploader.uploaded_files), 24) + for i, file_date in enumerate(sorted(uploader.uploaded_files)): + d = {'year': year, 'month': month, 'day': day, 'hour': i} + for k, v in d.items(): + d[k] = '%0.2d' % v + expected = (os.path.join(tmpdir, '%s.%s%0.2d.log' % + (i, today_str, i)), d) + # TODO: support wildcards before the date pattern + # (i.e. relative offsets) + #print file_date + #self.assertEquals(file_date, expected) + finally: + rmtree(tmpdir) if __name__ == '__main__': From 841a894955038cc54dd1b9b4d029e5390fc7c7f3 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 16 Feb 2011 19:52:22 -0600 Subject: [PATCH 62/64] pep8 --- test/unit/stats/test_log_uploader.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/test/unit/stats/test_log_uploader.py b/test/unit/stats/test_log_uploader.py index 145da9d710..b82e0ce02c 100644 --- a/test/unit/stats/test_log_uploader.py +++ b/test/unit/stats/test_log_uploader.py @@ -29,6 +29,7 @@ LOGGER = logging.getLogger() DEFAULT_GLOB = '%Y%m%d%H' + class TestLogUploader(unittest.TestCase): def test_upload_all_logs(self): @@ -42,13 +43,14 @@ class TestLogUploader(unittest.TestCase): def __init__(self, conf, logger=LOGGER): self.log_dir = conf['log_dir'] - self.filename_format = conf.get('filename_format', DEFAULT_GLOB) + self.filename_format = conf.get('filename_format', + DEFAULT_GLOB) self.new_log_cutoff = 0 self.logger = logger self.internal_proxy = MockInternalProxy() self.swift_account = '' self.container_name = '' - + self.uploaded_files = [] def upload_one_log(self, filename, year, month, day, hour): @@ -77,7 +79,8 @@ class TestLogUploader(unittest.TestCase): d = {'year': year, 'month': month, 'day': day, 'hour': i} for k, v in d.items(): d[k] = '%0.2d' % v - expected = (os.path.join(tmpdir, '%s%0.2d' % (today_str, i)), d) + expected = (os.path.join(tmpdir, '%s%0.2d' % + (today_str, i)), d) self.assertEquals(file_date, expected) finally: rmtree(tmpdir) @@ -94,11 +97,15 @@ class TestLogUploader(unittest.TestCase): for i in range(24): time_strs.append('%s-%0.2d00' % (today_str, i)) for ts in time_strs: - open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.gz' % ts), 'w').close() + open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.gz' % + ts), 'w').close() - open(os.path.join(tmpdir, 'swift.blah_98764.%s-2400.tar.gz' % ts), 'w').close() - open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.g' % ts), 'w').close() - open(os.path.join(tmpdir, 'swift-blah_201102160100.%s-2400.tar.gz' % + open(os.path.join(tmpdir, 'swift.blah_98764.%s-2400.tar.gz' % ts), + 'w').close() + open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.g' % ts), + 'w').close() + open(os.path.join(tmpdir, + 'swift-blah_201102160100.%s-2400.tar.gz' % '201102160100'), 'w').close() conf = { @@ -146,7 +153,7 @@ class TestLogUploader(unittest.TestCase): d = {'year': year, 'month': month, 'day': day, 'hour': i} for k, v in d.items(): d[k] = '%0.2d' % v - expected = (os.path.join(tmpdir, '%s.%s%0.2d.log' % + expected = (os.path.join(tmpdir, '%s.%s%0.2d.log' % (i, today_str, i)), d) # TODO: support wildcards before the date pattern # (i.e. relative offsets) From 13a34844707b4688f09c282893e5e36a3d70d18e Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Thu, 17 Feb 2011 08:30:39 +0000 Subject: [PATCH 63/64] rename log-processing to log-processor --- doc/source/overview_stats.rst | 2 +- etc/{log-processing.conf-sample => log-processor.conf-sample} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename etc/{log-processing.conf-sample => log-processor.conf-sample} (100%) diff --git a/doc/source/overview_stats.rst b/doc/source/overview_stats.rst index 6364de4611..111e1f8df0 100644 --- a/doc/source/overview_stats.rst +++ b/doc/source/overview_stats.rst @@ -181,4 +181,4 @@ earlier. This file will have one entry per account per hour for each account with activity in that hour. One .csv file should be produced per hour. Note that the stats will be delayed by at least two hours by default. This can be changed with the new_log_cutoff variable in the config file. See -`log-processing.conf-sample` for more details. \ No newline at end of file +`log-processor.conf-sample` for more details. diff --git a/etc/log-processing.conf-sample b/etc/log-processor.conf-sample similarity index 100% rename from etc/log-processing.conf-sample rename to etc/log-processor.conf-sample From ed69db162aa9daa84cc3c86aec0af0c3f840acf6 Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 17 Feb 2011 09:30:41 -0800 Subject: [PATCH 64/64] Revert wal+index --- swift/account/server.py | 7 + swift/common/db.py | 375 ++++++++++++++++++++++------------ swift/common/db_replicator.py | 6 +- swift/container/server.py | 4 + 4 files changed, 258 insertions(+), 134 deletions(-) diff --git a/swift/account/server.py b/swift/account/server.py index 79b840b501..f15ac38c11 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -86,6 +86,8 @@ class AccountController(object): return Response(status='507 %s is not mounted' % drive) broker = self._get_account_broker(drive, part, account) if container: # put account container + if 'x-cf-trans-id' in req.headers: + broker.pending_timeout = 3 if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): return HTTPNotFound(request=req) @@ -138,6 +140,9 @@ class AccountController(object): if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) broker = self._get_account_broker(drive, part, account) + if not container: + broker.pending_timeout = 0.1 + broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() @@ -166,6 +171,8 @@ class AccountController(object): if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) broker = self._get_account_broker(drive, part, account) + broker.pending_timeout = 0.1 + broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() diff --git a/swift/common/db.py b/swift/common/db.py index 9f322e7b7d..be96411619 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -27,14 +27,13 @@ import cPickle as pickle import errno from random import randint from tempfile import mkstemp -import traceback from eventlet import sleep import simplejson as json import sqlite3 from swift.common.utils import normalize_timestamp, renamer, \ - mkdirs, lock_parent_directory + mkdirs, lock_parent_directory, fallocate from swift.common.exceptions import LockTimeout @@ -42,9 +41,8 @@ from swift.common.exceptions import LockTimeout BROKER_TIMEOUT = 25 #: Pickle protocol to use PICKLE_PROTOCOL = 2 -CONNECT_ATTEMPTS = 4 -PENDING_COMMIT_TIMEOUT = 900 -AUTOCHECKPOINT = 8192 +#: Max number of pending entries +PENDING_CAP = 131072 class DatabaseConnectionError(sqlite3.DatabaseError): @@ -125,48 +123,48 @@ def get_db_connection(path, timeout=30, okay_to_create=False): :param okay_to_create: if True, create the DB if it doesn't exist :returns: DB connection object """ - # retry logic to address: - # http://www.mail-archive.com/sqlite-users@sqlite.org/msg57092.html - for attempt in xrange(CONNECT_ATTEMPTS): - try: - connect_time = time.time() - conn = sqlite3.connect(path, check_same_thread=False, - factory=GreenDBConnection, timeout=timeout) + try: + connect_time = time.time() + conn = sqlite3.connect(path, check_same_thread=False, + factory=GreenDBConnection, timeout=timeout) + if path != ':memory:' and not okay_to_create: # attempt to detect and fail when connect creates the db file - if path != ':memory:' and not okay_to_create: - stat = os.stat(path) - if stat.st_size == 0 and stat.st_ctime >= connect_time: - os.unlink(path) - raise DatabaseConnectionError(path, - 'DB file created by connect?') - conn.execute('PRAGMA journal_mode = WAL') - conn.execute('PRAGMA synchronous = NORMAL') - conn.execute('PRAGMA wal_autocheckpoint = %s' % AUTOCHECKPOINT) - conn.execute('PRAGMA count_changes = OFF') - conn.execute('PRAGMA temp_store = MEMORY') - conn.create_function('chexor', 3, chexor) - conn.row_factory = sqlite3.Row - conn.text_factory = str - return conn - except sqlite3.DatabaseError, e: - errstr = traceback.format_exc() - raise DatabaseConnectionError(path, errstr, timeout=timeout) + stat = os.stat(path) + if stat.st_size == 0 and stat.st_ctime >= connect_time: + os.unlink(path) + raise DatabaseConnectionError(path, + 'DB file created by connect?') + conn.row_factory = sqlite3.Row + conn.text_factory = str + conn.execute('PRAGMA synchronous = NORMAL') + conn.execute('PRAGMA count_changes = OFF') + conn.execute('PRAGMA temp_store = MEMORY') + conn.execute('PRAGMA journal_mode = DELETE') + conn.create_function('chexor', 3, chexor) + except sqlite3.DatabaseError: + import traceback + raise DatabaseConnectionError(path, traceback.format_exc(), + timeout=timeout) + return conn class DatabaseBroker(object): """Encapsulates working with a database.""" def __init__(self, db_file, timeout=BROKER_TIMEOUT, logger=None, - account=None, container=None): + account=None, container=None, pending_timeout=10, + stale_reads_ok=False): """ Encapsulates working with a database. """ self.conn = None self.db_file = db_file + self.pending_file = self.db_file + '.pending' + self.pending_timeout = pending_timeout + self.stale_reads_ok = stale_reads_ok self.db_dir = os.path.dirname(db_file) self.timeout = timeout self.logger = logger or logging.getLogger() self.account = account self.container = container - self._db_version = -1 def initialize(self, put_timestamp=None): """ @@ -235,7 +233,7 @@ class DatabaseBroker(object): conn.close() with open(tmp_db_file, 'r+b') as fp: os.fsync(fp.fileno()) - with lock_parent_directory(self.db_file, self.timeout): + with lock_parent_directory(self.db_file, self.pending_timeout): if os.path.exists(self.db_file): # It's as if there was a "condition" where different parts # of the system were "racing" each other. @@ -287,7 +285,6 @@ class DatabaseBroker(object): self.conn = None orig_isolation_level = conn.isolation_level conn.isolation_level = None - conn.execute('PRAGMA journal_mode = DELETE') # remove journal files conn.execute('BEGIN IMMEDIATE') try: yield True @@ -295,7 +292,6 @@ class DatabaseBroker(object): pass try: conn.execute('ROLLBACK') - conn.execute('PRAGMA journal_mode = WAL') # back to WAL mode conn.isolation_level = orig_isolation_level self.conn = conn except Exception: @@ -352,6 +348,11 @@ class DatabaseBroker(object): :param count: number to get :returns: list of objects between start and end """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise with self.get() as conn: curs = conn.execute(''' SELECT * FROM %s WHERE ROWID > ? ORDER BY ROWID ASC LIMIT ? @@ -400,7 +401,11 @@ class DatabaseBroker(object): :returns: dict containing keys: hash, id, created_at, put_timestamp, delete_timestamp, count, max_row, and metadata """ - self._commit_puts() + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise query_part1 = ''' SELECT hash, id, created_at, put_timestamp, delete_timestamp, %s_count AS count, @@ -450,6 +455,34 @@ class DatabaseBroker(object): (rec['sync_point'], rec['remote_id'])) conn.commit() + def _preallocate(self): + """ + The idea is to allocate space in front of an expanding db. If it gets + within 512k of a boundary, it allocates to the next boundary. + Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after. + """ + if self.db_file == ':memory:': + return + MB = (1024 * 1024) + + def prealloc_points(): + for pm in (1, 2, 5, 10, 25, 50): + yield pm * MB + while True: + pm += 50 + yield pm * MB + + stat = os.stat(self.db_file) + file_size = stat.st_size + allocated_size = stat.st_blocks * 512 + for point in prealloc_points(): + if file_size <= point - MB / 2: + prealloc_size = point + break + if allocated_size < prealloc_size: + with open(self.db_file, 'rb+') as fp: + fallocate(fp.fileno(), int(prealloc_size)) + @property def metadata(self): """ @@ -574,7 +607,7 @@ class ContainerBroker(DatabaseBroker): conn.executescript(""" CREATE TABLE object ( ROWID INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT, + name TEXT UNIQUE, created_at TEXT, size INTEGER, content_type TEXT, @@ -582,7 +615,7 @@ class ContainerBroker(DatabaseBroker): deleted INTEGER DEFAULT 0 ); - CREATE INDEX ix_object_deleted_name ON object (deleted, name); + CREATE INDEX ix_object_deleted ON object (deleted); CREATE TRIGGER object_insert AFTER INSERT ON object BEGIN @@ -645,15 +678,6 @@ class ContainerBroker(DatabaseBroker): ''', (self.account, self.container, normalize_timestamp(time.time()), str(uuid4()), put_timestamp)) - def _get_db_version(self, conn): - if self._db_version == -1: - self._db_version = 0 - for row in conn.execute(''' - SELECT name FROM sqlite_master - WHERE name = 'ix_object_deleted_name' '''): - self._db_version = 1 - return self._db_version - def _newid(self, conn): conn.execute(''' UPDATE container_stat @@ -693,6 +717,11 @@ class ContainerBroker(DatabaseBroker): :returns: True if the database has no active objects, False otherwise """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise with self.get() as conn: row = conn.execute( 'SELECT object_count from container_stat').fetchone() @@ -700,16 +729,17 @@ class ContainerBroker(DatabaseBroker): def _commit_puts(self, item_list=None): """Handles commiting rows in .pending files.""" - pending_file = self.db_file + '.pending' - if self.db_file == ':memory:' or not os.path.exists(pending_file): - return - if not os.path.getsize(pending_file): - os.unlink(pending_file) + if self.db_file == ':memory:' or not os.path.exists(self.pending_file): return if item_list is None: item_list = [] - with lock_parent_directory(pending_file, PENDING_COMMIT_TIMEOUT): - with open(pending_file, 'r+b') as fp: + with lock_parent_directory(self.pending_file, self.pending_timeout): + self._preallocate() + if not os.path.getsize(self.pending_file): + if item_list: + self.merge_items(item_list) + return + with open(self.pending_file, 'r+b') as fp: for entry in fp.read().split(':'): if entry: try: @@ -722,11 +752,11 @@ class ContainerBroker(DatabaseBroker): except Exception: self.logger.exception( _('Invalid pending entry %(file)s: %(entry)s'), - {'file': pending_file, 'entry': entry}) + {'file': self.pending_file, 'entry': entry}) if item_list: self.merge_items(item_list) try: - os.unlink(pending_file) + os.ftruncate(fp.fileno(), 0) except OSError, err: if err.errno != errno.ENOENT: raise @@ -744,6 +774,7 @@ class ContainerBroker(DatabaseBroker): delete :param sync_timestamp: max update_at timestamp of sync rows to delete """ + self._commit_puts() with self.get() as conn: conn.execute(""" DELETE FROM object @@ -787,9 +818,30 @@ class ContainerBroker(DatabaseBroker): record = {'name': name, 'created_at': timestamp, 'size': size, 'content_type': content_type, 'etag': etag, 'deleted': deleted} - if self.db_file != ':memory:' and not os.path.exists(self.db_file): + if self.db_file == ':memory:': + self.merge_items([record]) + return + if not os.path.exists(self.db_file): raise DatabaseConnectionError(self.db_file, "DB doesn't exist") - self.merge_items([record]) + pending_size = 0 + try: + pending_size = os.path.getsize(self.pending_file) + except OSError, err: + if err.errno != errno.ENOENT: + raise + if pending_size > PENDING_CAP: + self._commit_puts([record]) + else: + with lock_parent_directory( + self.pending_file, self.pending_timeout): + with open(self.pending_file, 'a+b') as fp: + # Colons aren't used in base64 encoding; so they are our + # delimiter + fp.write(':') + fp.write(pickle.dumps( + (name, timestamp, size, content_type, etag, deleted), + protocol=PICKLE_PROTOCOL).encode('base64')) + fp.flush() def is_deleted(self, timestamp=None): """ @@ -799,6 +851,11 @@ class ContainerBroker(DatabaseBroker): """ if self.db_file != ':memory:' and not os.path.exists(self.db_file): return True + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise with self.get() as conn: row = conn.execute(''' SELECT put_timestamp, delete_timestamp, object_count @@ -821,6 +878,11 @@ class ContainerBroker(DatabaseBroker): reported_put_timestamp, reported_delete_timestamp, reported_object_count, reported_bytes_used, hash, id) """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise with self.get() as conn: return conn.execute(''' SELECT account, container, created_at, put_timestamp, @@ -857,6 +919,11 @@ class ContainerBroker(DatabaseBroker): :returns: list of object names """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise rv = [] with self.get() as conn: row = conn.execute(''' @@ -893,6 +960,11 @@ class ContainerBroker(DatabaseBroker): :returns: list of tuples of (name, created_at, size, content_type, etag) """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise if path is not None: prefix = path if path: @@ -916,10 +988,7 @@ class ContainerBroker(DatabaseBroker): elif prefix: query += ' name >= ? AND' query_args.append(prefix) - if self._get_db_version(conn) < 1: - query += ' +deleted = 0 ORDER BY name LIMIT ?' - else: - query += ' deleted = 0 ORDER BY name LIMIT ?' + query += ' +deleted = 0 ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None @@ -967,19 +1036,18 @@ class ContainerBroker(DatabaseBroker): max_rowid = -1 for rec in item_list: conn.execute(''' - DELETE FROM object WHERE name = ? AND created_at < ? AND - deleted IN (0, 1) + DELETE FROM object WHERE name = ? AND + (created_at < ?) ''', (rec['name'], rec['created_at'])) - if not conn.execute(''' - SELECT name FROM object WHERE name = ? AND - deleted IN (0, 1) - ''', (rec['name'],)).fetchall(): + try: conn.execute(''' INSERT INTO object (name, created_at, size, content_type, etag, deleted) VALUES (?, ?, ?, ?, ?, ?) ''', ([rec['name'], rec['created_at'], rec['size'], rec['content_type'], rec['etag'], rec['deleted']])) + except sqlite3.IntegrityError: + pass if source: max_rowid = max(max_rowid, rec['ROWID']) if source: @@ -1023,7 +1091,7 @@ class AccountBroker(DatabaseBroker): conn.executescript(""" CREATE TABLE container ( ROWID INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT, + name TEXT UNIQUE, put_timestamp TEXT, delete_timestamp TEXT, object_count INTEGER, @@ -1031,9 +1099,8 @@ class AccountBroker(DatabaseBroker): deleted INTEGER DEFAULT 0 ); - CREATE INDEX ix_container_deleted_name ON - container (deleted, name); - + CREATE INDEX ix_container_deleted ON container (deleted); + CREATE INDEX ix_container_name ON container (name); CREATE TRIGGER container_insert AFTER INSERT ON container BEGIN UPDATE account_stat @@ -1097,15 +1164,6 @@ class AccountBroker(DatabaseBroker): ''', (self.account, normalize_timestamp(time.time()), str(uuid4()), put_timestamp)) - def _get_db_version(self, conn): - if self._db_version == -1: - self._db_version = 0 - for row in conn.execute(''' - SELECT name FROM sqlite_master - WHERE name = 'ix_container_deleted_name' '''): - self._db_version = 1 - return self._db_version - def update_put_timestamp(self, timestamp): """ Update the put_timestamp. Only modifies it if it is greater than @@ -1135,16 +1193,17 @@ class AccountBroker(DatabaseBroker): def _commit_puts(self, item_list=None): """Handles commiting rows in .pending files.""" - pending_file = self.db_file + '.pending' - if self.db_file == ':memory:' or not os.path.exists(pending_file): - return - if not os.path.getsize(pending_file): - os.unlink(pending_file) + if self.db_file == ':memory:' or not os.path.exists(self.pending_file): return if item_list is None: item_list = [] - with lock_parent_directory(pending_file, PENDING_COMMIT_TIMEOUT): - with open(pending_file, 'r+b') as fp: + with lock_parent_directory(self.pending_file, self.pending_timeout): + self._preallocate() + if not os.path.getsize(self.pending_file): + if item_list: + self.merge_items(item_list) + return + with open(self.pending_file, 'r+b') as fp: for entry in fp.read().split(':'): if entry: try: @@ -1160,11 +1219,11 @@ class AccountBroker(DatabaseBroker): except Exception: self.logger.exception( _('Invalid pending entry %(file)s: %(entry)s'), - {'file': pending_file, 'entry': entry}) + {'file': self.pending_file, 'entry': entry}) if item_list: self.merge_items(item_list) try: - os.unlink(pending_file) + os.ftruncate(fp.fileno(), 0) except OSError, err: if err.errno != errno.ENOENT: raise @@ -1175,6 +1234,11 @@ class AccountBroker(DatabaseBroker): :returns: True if the database has no active containers. """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise with self.get() as conn: row = conn.execute( 'SELECT container_count from account_stat').fetchone() @@ -1194,6 +1258,7 @@ class AccountBroker(DatabaseBroker): :param sync_timestamp: max update_at timestamp of sync rows to delete """ + self._commit_puts() with self.get() as conn: conn.execute(''' DELETE FROM container WHERE @@ -1221,6 +1286,11 @@ class AccountBroker(DatabaseBroker): :returns: put_timestamp of the container """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise with self.get() as conn: ret = conn.execute(''' SELECT put_timestamp FROM container @@ -1241,8 +1311,6 @@ class AccountBroker(DatabaseBroker): :param object_count: number of objects in the container :param bytes_used: number of bytes used by the container """ - if self.db_file != ':memory:' and not os.path.exists(self.db_file): - raise DatabaseConnectionError(self.db_file, "DB doesn't exist") if delete_timestamp > put_timestamp and \ object_count in (None, '', 0, '0'): deleted = 1 @@ -1253,7 +1321,24 @@ class AccountBroker(DatabaseBroker): 'object_count': object_count, 'bytes_used': bytes_used, 'deleted': deleted} - self.merge_items([record]) + if self.db_file == ':memory:': + self.merge_items([record]) + return + commit = False + with lock_parent_directory(self.pending_file, self.pending_timeout): + with open(self.pending_file, 'a+b') as fp: + # Colons aren't used in base64 encoding; so they are our + # delimiter + fp.write(':') + fp.write(pickle.dumps( + (name, put_timestamp, delete_timestamp, object_count, + bytes_used, deleted), + protocol=PICKLE_PROTOCOL).encode('base64')) + fp.flush() + if fp.tell() > PENDING_CAP: + commit = True + if commit: + self._commit_puts() def can_delete_db(self, cutoff): """ @@ -1261,6 +1346,7 @@ class AccountBroker(DatabaseBroker): :returns: True if the account can be deleted, False otherwise """ + self._commit_puts() with self.get() as conn: row = conn.execute(''' SELECT status, put_timestamp, delete_timestamp, container_count @@ -1286,6 +1372,11 @@ class AccountBroker(DatabaseBroker): """ if self.db_file != ':memory:' and not os.path.exists(self.db_file): return True + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise with self.get() as conn: row = conn.execute(''' SELECT put_timestamp, delete_timestamp, container_count, status @@ -1310,6 +1401,11 @@ class AccountBroker(DatabaseBroker): delete_timestamp, container_count, object_count, bytes_used, hash, id) """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise with self.get() as conn: return conn.execute(''' SELECT account, created_at, put_timestamp, delete_timestamp, @@ -1326,6 +1422,11 @@ class AccountBroker(DatabaseBroker): :returns: list of container names """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise rv = [] with self.get() as conn: row = conn.execute(''' @@ -1359,6 +1460,11 @@ class AccountBroker(DatabaseBroker): :returns: list of tuples of (name, object_count, bytes_used, 0) """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise if delimiter and not prefix: prefix = '' orig_marker = marker @@ -1379,10 +1485,7 @@ class AccountBroker(DatabaseBroker): elif prefix: query += ' name >= ? AND' query_args.append(prefix) - if self._get_db_version(conn) < 1: - query += ' +deleted = 0 ORDER BY name LIMIT ?' - else: - query += ' deleted = 0 ORDER BY name LIMIT ?' + query += ' +deleted = 0 ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None @@ -1426,39 +1529,51 @@ class AccountBroker(DatabaseBroker): record = [rec['name'], rec['put_timestamp'], rec['delete_timestamp'], rec['object_count'], rec['bytes_used'], rec['deleted']] - curs = conn.execute(''' - SELECT name, put_timestamp, delete_timestamp, - object_count, bytes_used, deleted - FROM container WHERE name = ? AND - deleted IN (0, 1) - ''', (rec['name'],)) - curs.row_factory = None - row = curs.fetchone() - if row: - row = list(row) - for i in xrange(5): - if record[i] is None and row[i] is not None: - record[i] = row[i] - if row[1] > record[1]: # Keep newest put_timestamp - record[1] = row[1] - if row[2] > record[2]: # Keep newest delete_timestamp - record[2] = row[2] - # If deleted, mark as such - if record[2] > record[1] and \ - record[3] in (None, '', 0, '0'): - record[5] = 1 - else: - record[5] = 0 - conn.execute(''' - DELETE FROM container WHERE name = ? AND - deleted IN (0, 1) - ''', (record[0],)) - conn.execute(''' - INSERT INTO container (name, put_timestamp, - delete_timestamp, object_count, bytes_used, - deleted) - VALUES (?, ?, ?, ?, ?, ?) - ''', record) + try: + conn.execute(''' + INSERT INTO container (name, put_timestamp, + delete_timestamp, object_count, bytes_used, + deleted) + VALUES (?, ?, ?, ?, ?, ?) + ''', record) + except sqlite3.IntegrityError: + curs = conn.execute(''' + SELECT name, put_timestamp, delete_timestamp, + object_count, bytes_used, deleted + FROM container WHERE name = ? AND + (put_timestamp < ? OR delete_timestamp < ? OR + object_count != ? OR bytes_used != ?)''', + (rec['name'], rec['put_timestamp'], + rec['delete_timestamp'], rec['object_count'], + rec['bytes_used'])) + curs.row_factory = None + row = curs.fetchone() + if row: + row = list(row) + for i in xrange(5): + if record[i] is None and row[i] is not None: + record[i] = row[i] + if row[1] > record[1]: # Keep newest put_timestamp + record[1] = row[1] + if row[2] > record[2]: # Keep newest delete_timestamp + record[2] = row[2] + conn.execute('DELETE FROM container WHERE name = ?', + (record[0],)) + # If deleted, mark as such + if record[2] > record[1] and \ + record[3] in (None, '', 0, '0'): + record[5] = 1 + else: + record[5] = 0 + try: + conn.execute(''' + INSERT INTO container (name, put_timestamp, + delete_timestamp, object_count, bytes_used, + deleted) + VALUES (?, ?, ?, ?, ?, ?) + ''', record) + except sqlite3.IntegrityError: + continue if source: max_rowid = max(max_rowid, rec['ROWID']) if source: diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 4b4b30fd30..3c3731d45a 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -180,9 +180,7 @@ class Replicator(Daemon): return False # perform block-level sync if the db was modified during the first sync if os.path.exists(broker.db_file + '-journal') or \ - os.path.exists(broker.db_file + '-wal') or \ - os.path.exists(broker.db_file + '-shm') or \ - os.path.getmtime(broker.db_file) > mtime: + os.path.getmtime(broker.db_file) > mtime: # grab a lock so nobody else can modify it with broker.lock(): if not self._rsync_file(broker.db_file, remote_file, False): @@ -318,7 +316,7 @@ class Replicator(Daemon): self.logger.debug(_('Replicating db %s'), object_file) self.stats['attempted'] += 1 try: - broker = self.brokerclass(object_file) + broker = self.brokerclass(object_file, pending_timeout=30) broker.reclaim(time.time() - self.reclaim_age, time.time() - (self.reclaim_age * 2)) info = broker.get_replication_info() diff --git a/swift/container/server.py b/swift/container/server.py index 1bfba49c36..549fc47596 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -219,6 +219,8 @@ class ContainerController(object): if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) broker = self._get_container_broker(drive, part, account, container) + broker.pending_timeout = 0.1 + broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() @@ -244,6 +246,8 @@ class ContainerController(object): if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) broker = self._get_container_broker(drive, part, account, container) + broker.pending_timeout = 0.1 + broker.stale_reads_ok = True if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info()