From 34f5085c3e143a27d9f3c4980b5b04514d78e659 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Mon, 25 Mar 2013 16:34:43 -0700 Subject: [PATCH] conf.d support Allow Swift daemons and servers to optionally accept a directory as the configuration parameter. Directory based configuration leverages ConfigParser's native multi-file support. Files ending in '.conf' in the given directory are parsed in lexicographical order. Filenames starting with '.' are ignored. A mixture of file and directory configuration paths is not supported - if the configuration path is a file behavior is unchanged. * update swift-init to search for conf.d paths when building servers (e.g. /etc/swift/proxy-server.conf.d/) * new script swift-config can be used to inspect the cumulative configuration * pull a little bit of code out of run_wsgi and test separately * fix example config bug for the proxy servers client_disconnect option * added section on directory based configuration to deployment guide DocImpact Implements: blueprint confd Change-Id: I89b0f48e538117f28590cf6698401f74ef58003b --- bin/swift-config | 58 ++++++++ doc/source/deployment_guide.rst | 96 ++++++++++++- etc/proxy-server.conf-sample | 2 +- setup.py | 1 + swift/common/manager.py | 18 +-- swift/common/utils.py | 68 ++++++--- swift/common/wsgi.py | 147 ++++++++++++++----- test/unit/common/test_manager.py | 41 ++++++ test/unit/common/test_utils.py | 101 ++++++++++++- test/unit/common/test_wsgi.py | 240 ++++++++++++++++++++++++++++++- 10 files changed, 699 insertions(+), 73 deletions(-) create mode 100755 bin/swift-config diff --git a/bin/swift-config b/bin/swift-config new file mode 100755 index 0000000000..25d91e984e --- /dev/null +++ b/bin/swift-config @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +import optparse +import os +import sys + +from swift.common.manager import Server +from swift.common.utils import readconf +from swift.common.wsgi import appconfig + +parser = optparse.OptionParser('%prog [options] SERVER') +parser.add_option('-c', '--config-num', metavar="N", type="int", + dest="number", default=0, + help="parse config for the Nth server only") +parser.add_option('-s', '--section', help="only display matching sections") +parser.add_option('-w', '--wsgi', action='store_true', + help="use wsgi/paste parser instead of readconf") + + +def main(): + options, args = parser.parse_args() + options = dict(vars(options)) + + if not args: + return 'ERROR: specify type of server or conf_path' + conf_files = [] + for arg in args: + if os.path.exists(arg): + conf_files.append(arg) + else: + conf_files += Server(arg).conf_files(**options) + for conf_file in conf_files: + print '# %s' % conf_file + if options['wsgi']: + app_config = appconfig(conf_file) + context = app_config.context + conf = dict([(c.name, c.config()) for c in getattr( + context, 'filter_contexts', [])]) + conf[context.name] = app_config + else: + conf = readconf(conf_file) + flat_vars = {} + for k, v in conf.items(): + if options['section'] and k != options['section']: + continue + if not isinstance(v, dict): + flat_vars[k] = v + continue + print '[%s]' % k + for opt, value in v.items(): + print '%s = %s' % (opt, value) + print + for k, v in flat_vars.items(): + print '# %s = %s' % (k, v) + print + +if __name__ == "__main__": + sys.exit(main()) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 4bf89e8fd5..c9f81854bf 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -139,15 +139,102 @@ swift-ring-builder with no options will display help text with available commands and options. More information on how the ring works internally can be found in the :doc:`Ring Overview `. +.. _general-service-configuration: + +----------------------------- +General Service Configuration +----------------------------- + +Most Swift services fall into two categories. Swift's wsgi servers and +background daemons. + +For more information specific to the configuration of Swift's wsgi servers +with paste deploy see :ref:`general-server-configuration` + +Configuration for servers and daemons can be expressed together in the same +file for each type of server, or separately. If a required section for the +service trying to start is missing there will be an error. The sections not +used by the service are ignored. + +Consider the example of an object storage node. By convention configuration +for the object-server, object-updater, object-replicator, and object-auditor +exist in a single file ``/etc/swift/object-server.conf``:: + + [DEFAULT] + + [pipeline:main] + pipeline = object-server + + [app:object-server] + use = egg:swift#object + + [object-replicator] + reclaim_age = 259200 + + [object-updater] + + [object-auditor] + +Swift services expect a configuration path as the first argument:: + + $ swift-object-auditor + Usage: swift-object-auditor CONFIG [options] + + Error: missing config path argument + +If you omit the object-auditor section this file could not be used as the +configuration path when starting the ``swift-object-auditor`` daemon:: + + $ swift-object-auditor /etc/swift/object-server.conf + Unable to find object-auditor config section in /etc/swift/object-server.conf + +If the configuration path is a directory instead of a file all of the files in +the directory with the file extension ".conf" will be combined to generate the +configuration object which is delivered to the Swift service. This is +referred to generally as "directory based configuration". + +Directory based configuration leverages ConfigParser's native multi-file +support. Files ending in ".conf" in the given directory are parsed in +lexicographical order. Filenames starting with '.' are ignored. A mixture of +file and directory configuration paths is not supported - if the configuration +path is a file only that file will be parsed. + +The swift service management tool ``swift-init`` has adopted the convention of +looking for ``/etc/swift/{type}-server.conf.d/`` if the file +``/etc/swift/{type}-server.conf`` file does not exist. + +When using directory based configuration, if the same option under the same +section appears more than once in different files, the last value parsed is +said to override previous occurrences. You can ensure proper override +precedence by prefixing the files in the configuration directory with +numerical values.:: + + /etc/swift/ + default.base + object-server.conf.d/ + 000_default.conf -> ../default.base + 001_default-override.conf + 010_server.conf + 020_replicator.conf + 030_updater.conf + 040_auditor.conf + +You can inspect the resulting combined configuration object using the +``swift-config`` command line tool + +.. _general-server-configuration: + ---------------------------- General Server Configuration ---------------------------- Swift uses paste.deploy (http://pythonpaste.org/deploy/) to manage server -configurations. Default configuration options are set in the `[DEFAULT]` -section, and any options specified there can be overridden in any of the other -sections BUT ONLY BY USING THE SYNTAX ``set option_name = value``. This is the -unfortunate way paste.deploy works and I'll try to explain it in full. +configurations. + +Default configuration options are set in the `[DEFAULT]` section, and any +options specified there can be overridden in any of the other sections BUT +ONLY BY USING THE SYNTAX ``set option_name = value``. This is the unfortunate +way paste.deploy works and I'll try to explain it in full. First, here's an example paste.deploy configuration file:: @@ -218,6 +305,7 @@ The main rule to remember when working with Swift configuration files is: configuration files. + --------------------------- Object Server Configuration --------------------------- diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 86107bf2eb..32fff87388 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -36,6 +36,7 @@ # log_statsd_metric_prefix = # Use a comma separated list of full url (http://foo.bar:1234,https://foo.bar) # cors_allow_origin = +# client_timeout = 60 # eventlet_debug = false # max_clients = 1024 @@ -55,7 +56,6 @@ use = egg:swift#proxy # object_chunk_size = 8192 # client_chunk_size = 8192 # node_timeout = 10 -# client_timeout = 60 # conn_timeout = 0.5 # How long without an error before a node's error count is reset. This will # also be how long before a node is reenabled after suppression is triggered. diff --git a/setup.py b/setup.py index 229dc97e18..29f486542a 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,7 @@ setup( 'bin/swift-account-server', 'bin/swift-bench', 'bin/swift-bench-client', + 'bin/swift-config', 'bin/swift-container-auditor', 'bin/swift-container-replicator', 'bin/swift-container-server', diff --git a/swift/common/manager.py b/swift/common/manager.py index f61a53fdcc..cb01430008 100644 --- a/swift/common/manager.py +++ b/swift/common/manager.py @@ -350,8 +350,8 @@ class Server(): """ return conf_file.replace( os.path.normpath(SWIFT_DIR), self.run_dir, 1).replace( - '%s-server' % self.type, self.server, 1).rsplit( - '.conf', 1)[0] + '.pid' + '%s-server' % self.type, self.server, 1).replace( + '.conf', '.pid', 1) def get_conf_file_name(self, pid_file): """Translate pid_file to a corresponding conf_file @@ -363,13 +363,13 @@ class Server(): """ if self.server in STANDALONE_SERVERS: return pid_file.replace( - os.path.normpath(self.run_dir), SWIFT_DIR, 1)\ - .rsplit('.pid', 1)[0] + '.conf' + os.path.normpath(self.run_dir), SWIFT_DIR, 1).replace( + '.pid', '.conf', 1) else: return pid_file.replace( os.path.normpath(self.run_dir), SWIFT_DIR, 1).replace( - self.server, '%s-server' % self.type, 1).rsplit( - '.pid', 1)[0] + '.conf' + self.server, '%s-server' % self.type, 1).replace( + '.pid', '.conf', 1) def conf_files(self, **kwargs): """Get conf files for this server @@ -380,10 +380,10 @@ class Server(): """ if self.server in STANDALONE_SERVERS: found_conf_files = search_tree(SWIFT_DIR, self.server + '*', - '.conf') + '.conf', dir_ext='.conf.d') else: found_conf_files = search_tree(SWIFT_DIR, '%s-server*' % self.type, - '.conf') + '.conf', dir_ext='.conf.d') number = kwargs.get('number') if number: try: @@ -412,7 +412,7 @@ class Server(): :returns: list of pid files """ - pid_files = search_tree(self.run_dir, '%s*' % self.server, '.pid') + pid_files = search_tree(self.run_dir, '%s*' % self.server) if kwargs.get('number', 0): conf_files = self.conf_files(**kwargs) # filter pid_files to match the index of numbered conf_file diff --git a/swift/common/utils.py b/swift/common/utils.py index 4853dd6d4a..12ab6f447c 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -941,7 +941,7 @@ def parse_options(parser=None, once=False, test_args=None): if not args: parser.print_usage() - print _("Error: missing config file argument") + print _("Error: missing config path argument") sys.exit(1) config = os.path.abspath(args.pop(0)) if not os.path.exists(config): @@ -1208,13 +1208,21 @@ def cache_from_env(env): return item_from_env(env, 'swift.cache') -def readconf(conffile, section_name=None, log_name=None, defaults=None, +def read_conf_dir(parser, conf_dir): + conf_files = [] + for f in os.listdir(conf_dir): + if f.endswith('.conf') and not f.startswith('.'): + conf_files.append(os.path.join(conf_dir, f)) + return parser.read(sorted(conf_files)) + + +def readconf(conf_path, section_name=None, log_name=None, defaults=None, raw=False): """ - Read config file and return config items as a dict + Read config file(s) and return config items as a dict - :param conffile: path to config file, or a file-like object (hasattr - readline) + :param conf_path: path to config file/directory, or a file-like object + (hasattr readline) :param section_name: config section to read (will return all sections if not defined) :param log_name: name to be used with logging (will use section_name if @@ -1228,18 +1236,23 @@ def readconf(conffile, section_name=None, log_name=None, defaults=None, c = RawConfigParser(defaults) else: c = ConfigParser(defaults) - if hasattr(conffile, 'readline'): - c.readfp(conffile) + if hasattr(conf_path, 'readline'): + c.readfp(conf_path) else: - if not c.read(conffile): - print _("Unable to read config file %s") % conffile + if os.path.isdir(conf_path): + # read all configs in directory + success = read_conf_dir(c, conf_path) + else: + success = c.read(conf_path) + if not success: + print _("Unable to read config from %s") % conf_path sys.exit(1) if section_name: if c.has_section(section_name): conf = dict(c.items(section_name)) else: print _("Unable to find %s config section in %s") % \ - (section_name, conffile) + (section_name, conf_path) sys.exit(1) if "log_name" not in conf: if log_name is not None: @@ -1252,7 +1265,7 @@ def readconf(conffile, section_name=None, log_name=None, defaults=None, conf.update({s: dict(c.items(s))}) if 'log_name' not in conf: conf['log_name'] = log_name - conf['__file__'] = conffile + conf['__file__'] = conf_path return conf @@ -1277,27 +1290,44 @@ def write_pickle(obj, dest, tmp=None, pickle_protocol=0): renamer(tmppath, dest) -def search_tree(root, glob_match, ext): - """Look in root, for any files/dirs matching glob, recurively traversing +def search_tree(root, glob_match, ext='', dir_ext=None): + """Look in root, for any files/dirs matching glob, recursively traversing any found directories looking for files ending with ext :param root: start of search path :param glob_match: glob to match in root, matching dirs are traversed with os.walk :param ext: only files that end in ext will be returned + :param dir_ext: if present directories that end with dir_ext will not be + traversed and instead will be returned as a matched path :returns: list of full paths to matching files, sorted """ found_files = [] for path in glob.glob(os.path.join(root, glob_match)): - if path.endswith(ext): - found_files.append(path) - else: + if os.path.isdir(path): for root, dirs, files in os.walk(path): - for file in files: - if file.endswith(ext): - found_files.append(os.path.join(root, file)) + if dir_ext and root.endswith(dir_ext): + found_files.append(root) + # the root is a config dir, descend no further + break + for file_ in files: + if ext and not file_.endswith(ext): + continue + found_files.append(os.path.join(root, file_)) + found_dir = False + for dir_ in dirs: + if dir_ext and dir_.endswith(dir_ext): + found_dir = True + found_files.append(os.path.join(root, dir_)) + if found_dir: + # do not descend further into matching directories + break + else: + if ext and not path.endswith(ext): + continue + found_files.append(path) return sorted(found_files) diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 239c2bd4a8..63e86777d4 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -26,7 +26,7 @@ from StringIO import StringIO import eventlet import eventlet.debug from eventlet import greenio, GreenPool, sleep, wsgi, listen -from paste.deploy import loadapp, appconfig +from paste.deploy import loadwsgi from eventlet.green import socket, ssl from urllib import unquote @@ -37,6 +37,74 @@ from swift.common.utils import capture_stdio, disable_fallocate, \ validate_configuration, get_hub +class NamedConfigLoader(loadwsgi.ConfigLoader): + """ + Patch paste.deploy's ConfigLoader so each context object will know what + config section it came from. + """ + + def get_context(self, object_type, name=None, global_conf=None): + context = super(NamedConfigLoader, self).get_context( + object_type, name=name, global_conf=global_conf) + context.name = name + return context + + +loadwsgi.ConfigLoader = NamedConfigLoader + + +class ConfigDirLoader(NamedConfigLoader): + """ + Read configuration from multiple files under the given path. + """ + + def __init__(self, conf_dir): + # parent class uses filename attribute when building error messages + self.filename = conf_dir = conf_dir.strip() + defaults = { + 'here': os.path.normpath(os.path.abspath(conf_dir)), + '__file__': os.path.abspath(conf_dir) + } + self.parser = loadwsgi.NicerConfigParser(conf_dir, defaults=defaults) + self.parser.optionxform = str # Don't lower-case keys + utils.read_conf_dir(self.parser, conf_dir) + + +def _loadconfigdir(object_type, uri, path, name, relative_to, global_conf): + if relative_to: + path = os.path.normpath(os.path.join(relative_to, path)) + loader = ConfigDirLoader(path) + if global_conf: + loader.update_defaults(global_conf, overwrite=False) + return loader.get_context(object_type, name, global_conf) + + +# add config_dir parsing to paste.deploy +loadwsgi._loaders['config_dir'] = _loadconfigdir + + +def wrap_conf_type(f): + """ + Wrap a function whos first argument is a paste.deploy style config uri, + such that you can pass it an un-adorned raw filesystem path and the config + directive (either config: or config_dir:) will be added automatically + based on the type of filesystem entity at the given path (either a file or + directory) before passing it through to the paste.deploy function. + """ + def wrapper(conf_path, *args, **kwargs): + if os.path.isdir(conf_path): + conf_type = 'config_dir' + else: + conf_type = 'config' + conf_uri = '%s:%s' % (conf_type, conf_path) + return f(conf_uri, *args, **kwargs) + return wrapper + + +appconfig = wrap_conf_type(loadwsgi.appconfig) +loadapp = wrap_conf_type(loadwsgi.loadapp) + + def monkey_patch_mimetools(): """ mimetools.Message defaults content-type to "text/plain" @@ -121,18 +189,47 @@ class RestrictedGreenPool(GreenPool): self.waitall() -# TODO: pull pieces of this out to test -def run_wsgi(conf_file, app_section, *args, **kwargs): +def run_server(conf, logger, sock): + wsgi.HttpProtocol.default_request_version = "HTTP/1.0" + # Turn off logging requests by the underlying WSGI software. + wsgi.HttpProtocol.log_request = lambda *a: None + # Redirect logging other messages by the underlying WSGI software. + wsgi.HttpProtocol.log_message = \ + lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a) + wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60) + + eventlet.hubs.use_hub(get_hub()) + eventlet.patcher.monkey_patch(all=False, socket=True) + eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no')) + eventlet.debug.hub_exceptions(eventlet_debug) + # utils.LogAdapter stashes name in server; fallback on unadapted loggers + if hasattr(logger, 'server'): + log_name = logger.server + else: + log_name = logger.name + app = loadapp(conf['__file__'], global_conf={'log_name': log_name}) + max_clients = int(conf.get('max_clients', '1024')) + pool = RestrictedGreenPool(size=max_clients) + try: + wsgi.server(sock, app, NullLogger(), custom_pool=pool) + except socket.error, err: + if err[0] != errno.EINVAL: + raise + pool.waitall() + + +# TODO: pull more pieces of this to test more +def run_wsgi(conf_path, app_section, *args, **kwargs): """ Runs the server using the specified number of workers. - :param conf_file: Path to paste.deploy style configuration file + :param conf_path: Path to paste.deploy style configuration file/directory :param app_section: App name from conf file to load config from """ # Load configuration, Set logger and Load request processor try: (app, conf, logger, log_name) = \ - init_request_processor(conf_file, app_section, *args, **kwargs) + init_request_processor(conf_path, app_section, *args, **kwargs) except ConfigFileError, e: print e return @@ -148,34 +245,10 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): # redirect errors to logger and close stdio capture_stdio(logger) - def run_server(max_clients): - wsgi.HttpProtocol.default_request_version = "HTTP/1.0" - # Turn off logging requests by the underlying WSGI software. - wsgi.HttpProtocol.log_request = lambda *a: None - # Redirect logging other messages by the underlying WSGI software. - wsgi.HttpProtocol.log_message = \ - lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a) - wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60) - - eventlet.hubs.use_hub(get_hub()) - eventlet.patcher.monkey_patch(all=False, socket=True) - eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no')) - eventlet.debug.hub_exceptions(eventlet_debug) - app = loadapp('config:%s' % conf_file, - global_conf={'log_name': log_name}) - pool = RestrictedGreenPool(size=max_clients) - try: - wsgi.server(sock, app, NullLogger(), custom_pool=pool) - except socket.error, err: - if err[0] != errno.EINVAL: - raise - pool.waitall() - - max_clients = int(conf.get('max_clients', '1024')) worker_count = int(conf.get('workers', '1')) # Useful for profiling [no forks]. if worker_count == 0: - run_server(max_clients) + run_server(conf, logger, sock) return def kill_children(*args): @@ -201,7 +274,7 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): if pid == 0: signal.signal(signal.SIGHUP, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) - run_server(max_clients) + run_server(conf, logger, sock) logger.notice('Child %d exiting normally' % os.getpid()) return else: @@ -227,22 +300,22 @@ class ConfigFileError(Exception): pass -def init_request_processor(conf_file, app_section, *args, **kwargs): +def init_request_processor(conf_path, app_section, *args, **kwargs): """ Loads common settings from conf Sets the logger Loads the request processor - :param conf_file: Path to paste.deploy style configuration file + :param conf_path: Path to paste.deploy style configuration file/directory :param app_section: App name from conf file to load config from :returns: the loaded application entry point :raises ConfigFileError: Exception is raised for config file error """ try: - conf = appconfig('config:%s' % conf_file, name=app_section) + conf = appconfig(conf_path, name=app_section) except Exception, e: - raise ConfigFileError("Error trying to load config %s: %s" % - (conf_file, e)) + raise ConfigFileError("Error trying to load config from %s: %s" % + (conf_path, e)) validate_configuration() @@ -260,7 +333,7 @@ def init_request_processor(conf_file, app_section, *args, **kwargs): disable_fallocate() monkey_patch_mimetools() - app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name}) + app = loadapp(conf_path, global_conf={'log_name': log_name}) return (app, conf, logger, log_name) diff --git a/test/unit/common/test_manager.py b/test/unit/common/test_manager.py index 8e2e5333c9..8ae0ef3a2e 100644 --- a/test/unit/common/test_manager.py +++ b/test/unit/common/test_manager.py @@ -453,6 +453,47 @@ class TestServer(unittest.TestCase): conf = self.join_swift_dir(server_name + '.conf') self.assertEquals(conf_file, conf) + def test_proxy_conf_dir(self): + conf_files = ( + 'proxy-server.conf.d/00.conf', + 'proxy-server.conf.d/01.conf', + ) + with temptree(conf_files) as t: + manager.SWIFT_DIR = t + server = manager.Server('proxy') + conf_dirs = server.conf_files() + self.assertEquals(len(conf_dirs), 1) + conf_dir = conf_dirs[0] + proxy_conf_dir = self.join_swift_dir('proxy-server.conf.d') + self.assertEquals(proxy_conf_dir, conf_dir) + + def test_conf_dir(self): + conf_files = ( + 'object-server/object-server.conf-base', + 'object-server/1.conf.d/base.conf', + 'object-server/1.conf.d/1.conf', + 'object-server/2.conf.d/base.conf', + 'object-server/2.conf.d/2.conf', + 'object-server/3.conf.d/base.conf', + 'object-server/3.conf.d/3.conf', + 'object-server/4.conf.d/base.conf', + 'object-server/4.conf.d/4.conf', + ) + with temptree(conf_files) as t: + manager.SWIFT_DIR = t + server = manager.Server('object-replicator') + conf_dirs = server.conf_files() + self.assertEquals(len(conf_dirs), 4) + c1 = self.join_swift_dir('object-server/1.conf.d') + c2 = self.join_swift_dir('object-server/2.conf.d') + c3 = self.join_swift_dir('object-server/3.conf.d') + c4 = self.join_swift_dir('object-server/4.conf.d') + for c in [c1, c2, c3, c4]: + self.assert_(c in conf_dirs) + # test configs returned sorted + sorted_confs = sorted([c1, c2, c3, c4]) + self.assertEquals(conf_dirs, sorted_confs) + def test_iter_pid_files(self): """ Server.iter_pid_files is kinda boring, test the diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 2f4e7319a4..0f5a0eb1bf 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -25,6 +25,7 @@ import random import re import socket import sys +from textwrap import dedent import time import unittest from threading import Thread @@ -366,7 +367,7 @@ class TestUtils(unittest.TestCase): utils.sys.stderr = stde self.assertRaises(SystemExit, utils.parse_options, once=True, test_args=[]) - self.assert_('missing config file' in stdo.getvalue()) + self.assert_('missing config' in stdo.getvalue()) # verify conf file must exist, context manager will delete temp file with NamedTemporaryFile() as f: @@ -721,6 +722,84 @@ log_name = %(yarr)s''' os.unlink('/tmp/test') self.assertRaises(SystemExit, utils.readconf, '/tmp/test') + def test_readconf_dir(self): + config_dir = { + 'server.conf.d/01.conf': """ + [DEFAULT] + port = 8080 + foo = bar + + [section1] + name=section1 + """, + 'server.conf.d/section2.conf': """ + [DEFAULT] + port = 8081 + bar = baz + + [section2] + name=section2 + """, + 'other-server.conf.d/01.conf': """ + [DEFAULT] + port = 8082 + + [section3] + name=section3 + """ + } + # strip indent from test config contents + config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) + with temptree(*zip(*config_dir.items())) as path: + conf_dir = os.path.join(path, 'server.conf.d') + conf = utils.readconf(conf_dir) + expected = { + '__file__': os.path.join(path, 'server.conf.d'), + 'log_name': None, + 'section1': { + 'port': '8081', + 'foo': 'bar', + 'bar': 'baz', + 'name': 'section1', + }, + 'section2': { + 'port': '8081', + 'foo': 'bar', + 'bar': 'baz', + 'name': 'section2', + }, + } + self.assertEquals(conf, expected) + + def test_readconf_dir_ignores_hidden_and_nondotconf_files(self): + config_dir = { + 'server.conf.d/01.conf': """ + [section1] + port = 8080 + """, + 'server.conf.d/.01.conf.swp': """ + [section] + port = 8081 + """, + 'server.conf.d/01.conf-bak': """ + [section] + port = 8082 + """, + } + # strip indent from test config contents + config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) + with temptree(*zip(*config_dir.items())) as path: + conf_dir = os.path.join(path, 'server.conf.d') + conf = utils.readconf(conf_dir) + expected = { + '__file__': os.path.join(path, 'server.conf.d'), + 'log_name': None, + 'section1': { + 'port': '8080', + }, + } + self.assertEquals(conf, expected) + def test_drop_privileges(self): user = getuser() # over-ride os with mock @@ -925,6 +1004,26 @@ log_name = %(yarr)s''' for f in [f1, f2, f3, f4]: self.assert_(f in folder_texts) + def test_search_tree_with_directory_ext_match(self): + files = ( + 'object-server/object-server.conf-base', + 'object-server/1.conf.d/base.conf', + 'object-server/1.conf.d/1.conf', + 'object-server/2.conf.d/base.conf', + 'object-server/2.conf.d/2.conf', + 'object-server/3.conf.d/base.conf', + 'object-server/3.conf.d/3.conf', + 'object-server/4.conf.d/base.conf', + 'object-server/4.conf.d/4.conf', + ) + with temptree(files) as t: + conf_dirs = utils.search_tree(t, 'object-server', '.conf', + dir_ext='conf.d') + self.assertEquals(len(conf_dirs), 4) + for i in range(4): + conf_dir = os.path.join(t, 'object-server/%d.conf.d' % (i + 1)) + self.assert_(conf_dir in conf_dirs) + def test_write_file(self): with temptree([]) as t: file_name = os.path.join(t, 'test') diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index a30c21a3c7..3c96f16470 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -13,24 +13,64 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" Tests for swift.common.utils """ +""" Tests for swift.common.wsgi """ from __future__ import with_statement import errno +import logging import mimetools import socket import unittest +import os +import pickle +from textwrap import dedent +from gzip import GzipFile from StringIO import StringIO from collections import defaultdict from urllib import quote +from eventlet import listen + +import swift from swift.common.swob import Request -from swift.common import wsgi +from swift.common import wsgi, utils, ring + +from test.unit import temptree + +from mock import patch + + +def _fake_rings(tmpdir): + pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', + 'port': 6012}, + {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', + 'port': 6022}], 30), + GzipFile(os.path.join(tmpdir, 'account.ring.gz'), 'wb')) + pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', + 'port': 6011}, + {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', + 'port': 6021}], 30), + GzipFile(os.path.join(tmpdir, 'container.ring.gz'), 'wb')) + pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', + 'port': 6010}, + {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', + 'port': 6020}], 30), + GzipFile(os.path.join(tmpdir, 'object.ring.gz'), 'wb')) class TestWSGI(unittest.TestCase): """ Tests for swift.common.wsgi """ + def setUp(self): + utils.HASH_PATH_PREFIX = 'startcap' + self._orig_parsetype = mimetools.Message.parsetype + + def tearDown(self): + mimetools.Message.parsetype = self._orig_parsetype + def test_monkey_patch_mimetools(self): sio = StringIO('blah') self.assertEquals(mimetools.Message(sio).type, 'text/plain') @@ -69,6 +109,90 @@ class TestWSGI(unittest.TestCase): sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') self.assertEquals(mimetools.Message(sio).subtype, 'html') + def test_init_request_processor(self): + config = """ + [DEFAULT] + swift_dir = TEMPDIR + + [pipeline:main] + pipeline = catch_errors proxy-server + + [app:proxy-server] + use = egg:swift#proxy + conn_timeout = 0.2 + + [filter:catch_errors] + use = egg:swift#catch_errors + """ + contents = dedent(config) + with temptree(['proxy-server.conf']) as t: + conf_file = os.path.join(t, 'proxy-server.conf') + with open(conf_file, 'w') as f: + f.write(contents.replace('TEMPDIR', t)) + _fake_rings(t) + app, conf, logger, log_name = wsgi.init_request_processor( + conf_file, 'proxy-server') + # verify pipeline is catch_errors -> proxy-servery + expected = swift.common.middleware.catch_errors.CatchErrorMiddleware + self.assert_(isinstance(app, expected)) + self.assert_(isinstance(app.app, swift.proxy.server.Application)) + # config settings applied to app instance + self.assertEquals(0.2, app.app.conn_timeout) + # appconfig returns values from 'proxy-server' section + expected = { + '__file__': conf_file, + 'here': os.path.dirname(conf_file), + 'conn_timeout': '0.2', + 'swift_dir': t, + } + self.assertEquals(expected, conf) + # logger works + logger.info('testing') + self.assertEquals('proxy-server', log_name) + + def test_init_request_processor_from_conf_dir(self): + config_dir = { + 'proxy-server.conf.d/pipeline.conf': """ + [pipeline:main] + pipeline = catch_errors proxy-server + """, + 'proxy-server.conf.d/app.conf': """ + [app:proxy-server] + use = egg:swift#proxy + conn_timeout = 0.2 + """, + 'proxy-server.conf.d/catch-errors.conf': """ + [filter:catch_errors] + use = egg:swift#catch_errors + """ + } + # strip indent from test config contents + config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) + with temptree(*zip(*config_dir.items())) as conf_root: + conf_dir = os.path.join(conf_root, 'proxy-server.conf.d') + with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f: + f.write('[DEFAULT]\nswift_dir = %s' % conf_root) + _fake_rings(conf_root) + app, conf, logger, log_name = wsgi.init_request_processor( + conf_dir, 'proxy-server') + # verify pipeline is catch_errors -> proxy-servery + expected = swift.common.middleware.catch_errors.CatchErrorMiddleware + self.assert_(isinstance(app, expected)) + self.assert_(isinstance(app.app, swift.proxy.server.Application)) + # config settings applied to app instance + self.assertEquals(0.2, app.app.conn_timeout) + # appconfig returns values from 'proxy-server' section + expected = { + '__file__': conf_dir, + 'here': conf_dir, + 'conn_timeout': '0.2', + 'swift_dir': conf_root, + } + self.assertEquals(expected, conf) + # logger works + logger.info('testing') + self.assertEquals('proxy-server', log_name) + def test_get_socket(self): # stubs conf = {} @@ -170,6 +294,117 @@ class TestWSGI(unittest.TestCase): wsgi.sleep = old_sleep wsgi.time = old_time + def test_run_server(self): + config = """ + [DEFAULT] + eventlet_debug = yes + client_timeout = 30 + swift_dir = TEMPDIR + + [pipeline:main] + pipeline = proxy-server + + [app:proxy-server] + use = egg:swift#proxy + """ + + contents = dedent(config) + with temptree(['proxy-server.conf']) as t: + conf_file = os.path.join(t, 'proxy-server.conf') + with open(conf_file, 'w') as f: + f.write(contents.replace('TEMPDIR', t)) + _fake_rings(t) + with patch('swift.common.wsgi.wsgi') as _wsgi: + with patch('swift.common.wsgi.eventlet') as _eventlet: + conf = wsgi.appconfig(conf_file) + logger = logging.getLogger('test') + sock = listen(('localhost', 0)) + wsgi.run_server(conf, logger, sock) + self.assertEquals('HTTP/1.0', + _wsgi.HttpProtocol.default_request_version) + self.assertEquals(30, _wsgi.WRITE_TIMEOUT) + _eventlet.hubs.use_hub.assert_called_with(utils.get_hub()) + _eventlet.patcher.monkey_patch.assert_called_with(all=False, + socket=True) + _eventlet.debug.hub_exceptions.assert_called_with(True) + _wsgi.server.assert_called() + args, kwargs = _wsgi.server.call_args + server_sock, server_app, server_logger = args + self.assertEquals(sock, server_sock) + self.assert_(isinstance(server_app, swift.proxy.server.Application)) + self.assert_(isinstance(server_logger, wsgi.NullLogger)) + self.assert_('custom_pool' in kwargs) + + def test_run_server_conf_dir(self): + config_dir = { + 'proxy-server.conf.d/pipeline.conf': """ + [pipeline:main] + pipeline = proxy-server + """, + 'proxy-server.conf.d/app.conf': """ + [app:proxy-server] + use = egg:swift#proxy + """, + 'proxy-server.conf.d/default.conf': """ + [DEFAULT] + eventlet_debug = yes + client_timeout = 30 + """ + } + # strip indent from test config contents + config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) + with temptree(*zip(*config_dir.items())) as conf_root: + conf_dir = os.path.join(conf_root, 'proxy-server.conf.d') + with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f: + f.write('[DEFAULT]\nswift_dir = %s' % conf_root) + _fake_rings(conf_root) + with patch('swift.common.wsgi.wsgi') as _wsgi: + with patch('swift.common.wsgi.eventlet') as _eventlet: + conf = wsgi.appconfig(conf_dir) + logger = logging.getLogger('test') + sock = listen(('localhost', 0)) + wsgi.run_server(conf, logger, sock) + + self.assertEquals('HTTP/1.0', + _wsgi.HttpProtocol.default_request_version) + self.assertEquals(30, _wsgi.WRITE_TIMEOUT) + _eventlet.hubs.use_hub.assert_called_with(utils.get_hub()) + _eventlet.patcher.monkey_patch.assert_called_with(all=False, + socket=True) + _eventlet.debug.hub_exceptions.assert_called_with(True) + _wsgi.server.assert_called() + args, kwargs = _wsgi.server.call_args + server_sock, server_app, server_logger = args + self.assertEquals(sock, server_sock) + self.assert_(isinstance(server_app, swift.proxy.server.Application)) + self.assert_(isinstance(server_logger, wsgi.NullLogger)) + self.assert_('custom_pool' in kwargs) + + def test_appconfig_dir_ignores_hidden_files(self): + config_dir = { + 'server.conf.d/01.conf': """ + [app:main] + use = egg:swift#proxy + port = 8080 + """, + 'server.conf.d/.01.conf.swp': """ + [app:main] + use = egg:swift#proxy + port = 8081 + """, + } + # strip indent from test config contents + config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) + with temptree(*zip(*config_dir.items())) as path: + conf_dir = os.path.join(path, 'server.conf.d') + conf = wsgi.appconfig(conf_dir) + expected = { + '__file__': os.path.join(path, 'server.conf.d'), + 'here': os.path.join(path, 'server.conf.d'), + 'port': '8080', + } + self.assertEquals(conf, expected) + def test_pre_auth_wsgi_input(self): oldenv = {} newenv = wsgi.make_pre_authed_env(oldenv) @@ -246,6 +481,7 @@ class TestWSGI(unittest.TestCase): self.assertEquals(r.body, 'the body') self.assertEquals(r.environ['swift.source'], 'UT') + class TestWSGIContext(unittest.TestCase): def test_app_call(self):