openstack
/
swift
Code Issues Proposed changes

conf.d support 

Allow Swift daemons and servers to optionally accept a directory as the
configuration parameter.  Directory based configuration leverages
ConfigParser's native multi-file support.  Files ending in '.conf' in the
given directory are parsed in lexicographical order.  Filenames starting with
'.' are ignored.  A mixture of file and directory configuration paths is not
supported - if the configuration path is a file behavior is unchanged.

 * update swift-init to search for conf.d paths when building servers
    (e.g. /etc/swift/proxy-server.conf.d/)
 * new script swift-config can be used to inspect the cumulative configuration
 * pull a little bit of code out of run_wsgi and test separately
 * fix example config bug for the proxy servers client_disconnect option
 * added section on directory based configuration to deployment guide

DocImpact

Implements: blueprint confd
Change-Id: I89b0f48e538117f28590cf6698401f74ef58003b
This commit is contained in:
Clay Gerrard 2013-03-25 16:34:43 -07:00
parent 52a6595033
commit 34f5085c3e
10 changed files with 699 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
bin/swift-config Executable file
View File

@ -0,0 +1,58 @@
#!/usr/bin/env python
import optparse
import os
import sys
from swift.common.manager import Server
from swift.common.utils import readconf
from swift.common.wsgi import appconfig
parser = optparse.OptionParser('%prog [options] SERVER')
parser.add_option('-c', '--config-num', metavar="N", type="int",
dest="number", default=0,
help="parse config for the Nth server only")
parser.add_option('-s', '--section', help="only display matching sections")
parser.add_option('-w', '--wsgi', action='store_true',
help="use wsgi/paste parser instead of readconf")
def main():
options, args = parser.parse_args()
options = dict(vars(options))
if not args:
return 'ERROR: specify type of server or conf_path'
conf_files = []
for arg in args:
if os.path.exists(arg):
conf_files.append(arg)
else:
conf_files += Server(arg).conf_files(**options)
for conf_file in conf_files:
print '# %s' % conf_file
if options['wsgi']:
app_config = appconfig(conf_file)
context = app_config.context
conf = dict([(c.name, c.config()) for c in getattr(
context, 'filter_contexts', [])])
conf[context.name] = app_config
else:
conf = readconf(conf_file)
flat_vars = {}
for k, v in conf.items():
if options['section'] and k != options['section']:
continue
if not isinstance(v, dict):
flat_vars[k] = v
continue
print '[%s]' % k
for opt, value in v.items():
print '%s = %s' % (opt, value)
print
for k, v in flat_vars.items():
print '# %s = %s' % (k, v)
print
if __name__ == "__main__":
sys.exit(main())

96
doc/source/deployment_guide.rst
View File

@ -139,15 +139,102 @@ swift-ring-builder with no options will display help text with available
commands and options. More information on how the ring works internally
can be found in the :doc:`Ring Overview <overview_ring>`.
.. _general-service-configuration:
-----------------------------
General Service Configuration
-----------------------------
Most Swift services fall into two categories. Swift's wsgi servers and
background daemons.
For more information specific to the configuration of Swift's wsgi servers
with paste deploy see :ref:`general-server-configuration`
Configuration for servers and daemons can be expressed together in the same
file for each type of server, or separately. If a required section for the
service trying to start is missing there will be an error. The sections not
used by the service are ignored.
Consider the example of an object storage node. By convention configuration
for the object-server, object-updater, object-replicator, and object-auditor
exist in a single file ``/etc/swift/object-server.conf``::
[DEFAULT]
[pipeline:main]
pipeline = object-server
[app:object-server]
use = egg:swift#object
[object-replicator]
reclaim_age = 259200
[object-updater]
[object-auditor]
Swift services expect a configuration path as the first argument::
$ swift-object-auditor
Usage: swift-object-auditor CONFIG [options]
Error: missing config path argument
If you omit the object-auditor section this file could not be used as the
configuration path when starting the ``swift-object-auditor`` daemon::
$ swift-object-auditor /etc/swift/object-server.conf
Unable to find object-auditor config section in /etc/swift/object-server.conf
If the configuration path is a directory instead of a file all of the files in
the directory with the file extension ".conf" will be combined to generate the
configuration object which is delivered to the Swift service. This is
referred to generally as "directory based configuration".
Directory based configuration leverages ConfigParser's native multi-file
support. Files ending in ".conf" in the given directory are parsed in
lexicographical order. Filenames starting with '.' are ignored. A mixture of
file and directory configuration paths is not supported - if the configuration
path is a file only that file will be parsed.
The swift service management tool ``swift-init`` has adopted the convention of
looking for ``/etc/swift/{type}-server.conf.d/`` if the file
``/etc/swift/{type}-server.conf`` file does not exist.
When using directory based configuration, if the same option under the same
section appears more than once in different files, the last value parsed is
said to override previous occurrences. You can ensure proper override
precedence by prefixing the files in the configuration directory with
numerical values.::
/etc/swift/
default.base
object-server.conf.d/
000_default.conf -> ../default.base
001_default-override.conf
010_server.conf
020_replicator.conf
030_updater.conf
040_auditor.conf
You can inspect the resulting combined configuration object using the
``swift-config`` command line tool
.. _general-server-configuration:
----------------------------
General Server Configuration
----------------------------
Swift uses paste.deploy (http://pythonpaste.org/deploy/) to manage server
configurations. Default configuration options are set in the `[DEFAULT]`
section, and any options specified there can be overridden in any of the other
sections BUT ONLY BY USING THE SYNTAX ``set option_name = value``. This is the
unfortunate way paste.deploy works and I'll try to explain it in full.
configurations.
Default configuration options are set in the `[DEFAULT]` section, and any
options specified there can be overridden in any of the other sections BUT
ONLY BY USING THE SYNTAX ``set option_name = value``. This is the unfortunate
way paste.deploy works and I'll try to explain it in full.
First, here's an example paste.deploy configuration file::
@ -218,6 +305,7 @@ The main rule to remember when working with Swift configuration files is:
configuration files.
---------------------------
Object Server Configuration
---------------------------

2
etc/proxy-server.conf-sample
View File

@ -36,6 +36,7 @@
# log_statsd_metric_prefix =
# Use a comma separated list of full url (http://foo.bar:1234,https://foo.bar)
# cors_allow_origin =
# client_timeout = 60
# eventlet_debug = false
# max_clients = 1024
@ -55,7 +56,6 @@ use = egg:swift#proxy
# object_chunk_size = 8192
# client_chunk_size = 8192
# node_timeout = 10
# client_timeout = 60
# conn_timeout = 0.5
# How long without an error before a node's error count is reset. This will
# also be how long before a node is reenabled after suppression is triggered.

1
setup.py
View File

@ -55,6 +55,7 @@ setup(
'bin/swift-account-server',
'bin/swift-bench',
'bin/swift-bench-client',
'bin/swift-config',
'bin/swift-container-auditor',
'bin/swift-container-replicator',
'bin/swift-container-server',

18
swift/common/manager.py
View File

@ -350,8 +350,8 @@ class Server():
"""
return conf_file.replace(
os.path.normpath(SWIFT_DIR), self.run_dir, 1).replace(
'%s-server' % self.type, self.server, 1).rsplit(
'.conf', 1)[0] + '.pid'
'%s-server' % self.type, self.server, 1).replace(
'.conf', '.pid', 1)
def get_conf_file_name(self, pid_file):
"""Translate pid_file to a corresponding conf_file
@ -363,13 +363,13 @@ class Server():
"""
if self.server in STANDALONE_SERVERS:
return pid_file.replace(
os.path.normpath(self.run_dir), SWIFT_DIR, 1)\
.rsplit('.pid', 1)[0] + '.conf'
os.path.normpath(self.run_dir), SWIFT_DIR, 1).replace(
'.pid', '.conf', 1)
else:
return pid_file.replace(
os.path.normpath(self.run_dir), SWIFT_DIR, 1).replace(
self.server, '%s-server' % self.type, 1).rsplit(
'.pid', 1)[0] + '.conf'
self.server, '%s-server' % self.type, 1).replace(
'.pid', '.conf', 1)
def conf_files(self, **kwargs):
"""Get conf files for this server
@ -380,10 +380,10 @@ class Server():
"""
if self.server in STANDALONE_SERVERS:
found_conf_files = search_tree(SWIFT_DIR, self.server + '*',
'.conf')
'.conf', dir_ext='.conf.d')
else:
found_conf_files = search_tree(SWIFT_DIR, '%s-server*' % self.type,
'.conf')
'.conf', dir_ext='.conf.d')
number = kwargs.get('number')
if number:
try:
@ -412,7 +412,7 @@ class Server():
:returns: list of pid files
"""
pid_files = search_tree(self.run_dir, '%s*' % self.server, '.pid')
pid_files = search_tree(self.run_dir, '%s*' % self.server)
if kwargs.get('number', 0):
conf_files = self.conf_files(**kwargs)
# filter pid_files to match the index of numbered conf_file

68
swift/common/utils.py
View File

@ -941,7 +941,7 @@ def parse_options(parser=None, once=False, test_args=None):
if not args:
parser.print_usage()
print _("Error: missing config file argument")
print _("Error: missing config path argument")
sys.exit(1)
config = os.path.abspath(args.pop(0))
if not os.path.exists(config):
@ -1208,13 +1208,21 @@ def cache_from_env(env):
return item_from_env(env, 'swift.cache')
def readconf(conffile, section_name=None, log_name=None, defaults=None,
def read_conf_dir(parser, conf_dir):
conf_files = []
for f in os.listdir(conf_dir):
if f.endswith('.conf') and not f.startswith('.'):
conf_files.append(os.path.join(conf_dir, f))
return parser.read(sorted(conf_files))
def readconf(conf_path, section_name=None, log_name=None, defaults=None,
raw=False):
"""
Read config file and return config items as a dict
Read config file(s) and return config items as a dict
:param conffile: path to config file, or a file-like object (hasattr
readline)
:param conf_path: path to config file/directory, or a file-like object
(hasattr readline)
:param section_name: config section to read (will return all sections if
not defined)
:param log_name: name to be used with logging (will use section_name if
@ -1228,18 +1236,23 @@ def readconf(conffile, section_name=None, log_name=None, defaults=None,
c = RawConfigParser(defaults)
else:
c = ConfigParser(defaults)
if hasattr(conffile, 'readline'):
c.readfp(conffile)
if hasattr(conf_path, 'readline'):
c.readfp(conf_path)
else:
if not c.read(conffile):
print _("Unable to read config file %s") % conffile
if os.path.isdir(conf_path):
# read all configs in directory
success = read_conf_dir(c, conf_path)
else:
success = c.read(conf_path)
if not success:
print _("Unable to read config from %s") % conf_path
sys.exit(1)
if section_name:
if c.has_section(section_name):
conf = dict(c.items(section_name))
else:
print _("Unable to find %s config section in %s") % \
(section_name, conffile)
(section_name, conf_path)
sys.exit(1)
if "log_name" not in conf:
if log_name is not None:
@ -1252,7 +1265,7 @@ def readconf(conffile, section_name=None, log_name=None, defaults=None,
conf.update({s: dict(c.items(s))})
if 'log_name' not in conf:
conf['log_name'] = log_name
conf['__file__'] = conffile
conf['__file__'] = conf_path
return conf
@ -1277,27 +1290,44 @@ def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
renamer(tmppath, dest)
def search_tree(root, glob_match, ext):
"""Look in root, for any files/dirs matching glob, recurively traversing
def search_tree(root, glob_match, ext='', dir_ext=None):
"""Look in root, for any files/dirs matching glob, recursively traversing
any found directories looking for files ending with ext
:param root: start of search path
:param glob_match: glob to match in root, matching dirs are traversed with
os.walk
:param ext: only files that end in ext will be returned
:param dir_ext: if present directories that end with dir_ext will not be
traversed and instead will be returned as a matched path
:returns: list of full paths to matching files, sorted
"""
found_files = []
for path in glob.glob(os.path.join(root, glob_match)):
if path.endswith(ext):
found_files.append(path)
else:
if os.path.isdir(path):
for root, dirs, files in os.walk(path):
for file in files:
if file.endswith(ext):
found_files.append(os.path.join(root, file))
if dir_ext and root.endswith(dir_ext):
found_files.append(root)
# the root is a config dir, descend no further
break
for file_ in files:
if ext and not file_.endswith(ext):
continue
found_files.append(os.path.join(root, file_))
found_dir = False
for dir_ in dirs:
if dir_ext and dir_.endswith(dir_ext):
found_dir = True
found_files.append(os.path.join(root, dir_))
if found_dir:
# do not descend further into matching directories
break
else:
if ext and not path.endswith(ext):
continue
found_files.append(path)
return sorted(found_files)

147
swift/common/wsgi.py
View File

@ -26,7 +26,7 @@ from StringIO import StringIO
import eventlet
import eventlet.debug
from eventlet import greenio, GreenPool, sleep, wsgi, listen
from paste.deploy import loadapp, appconfig
from paste.deploy import loadwsgi
from eventlet.green import socket, ssl
from urllib import unquote
@ -37,6 +37,74 @@ from swift.common.utils import capture_stdio, disable_fallocate, \
validate_configuration, get_hub
class NamedConfigLoader(loadwsgi.ConfigLoader):
"""
Patch paste.deploy's ConfigLoader so each context object will know what
config section it came from.
"""
def get_context(self, object_type, name=None, global_conf=None):
context = super(NamedConfigLoader, self).get_context(
object_type, name=name, global_conf=global_conf)
context.name = name
return context
loadwsgi.ConfigLoader = NamedConfigLoader
class ConfigDirLoader(NamedConfigLoader):
"""
Read configuration from multiple files under the given path.
"""
def __init__(self, conf_dir):
# parent class uses filename attribute when building error messages
self.filename = conf_dir = conf_dir.strip()
defaults = {
'here': os.path.normpath(os.path.abspath(conf_dir)),
'__file__': os.path.abspath(conf_dir)
}
self.parser = loadwsgi.NicerConfigParser(conf_dir, defaults=defaults)
self.parser.optionxform = str # Don't lower-case keys
utils.read_conf_dir(self.parser, conf_dir)
def _loadconfigdir(object_type, uri, path, name, relative_to, global_conf):
if relative_to:
path = os.path.normpath(os.path.join(relative_to, path))
loader = ConfigDirLoader(path)
if global_conf:
loader.update_defaults(global_conf, overwrite=False)
return loader.get_context(object_type, name, global_conf)
# add config_dir parsing to paste.deploy
loadwsgi._loaders['config_dir'] = _loadconfigdir
def wrap_conf_type(f):
"""
Wrap a function whos first argument is a paste.deploy style config uri,
such that you can pass it an un-adorned raw filesystem path and the config
directive (either config: or config_dir:) will be added automatically
based on the type of filesystem entity at the given path (either a file or
directory) before passing it through to the paste.deploy function.
"""
def wrapper(conf_path, *args, **kwargs):
if os.path.isdir(conf_path):
conf_type = 'config_dir'
else:
conf_type = 'config'
conf_uri = '%s:%s' % (conf_type, conf_path)
return f(conf_uri, *args, **kwargs)
return wrapper
appconfig = wrap_conf_type(loadwsgi.appconfig)
loadapp = wrap_conf_type(loadwsgi.loadapp)
def monkey_patch_mimetools():
"""
mimetools.Message defaults content-type to "text/plain"
@ -121,18 +189,47 @@ class RestrictedGreenPool(GreenPool):
self.waitall()
# TODO: pull pieces of this out to test
def run_wsgi(conf_file, app_section, *args, **kwargs):
def run_server(conf, logger, sock):
wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
# Turn off logging requests by the underlying WSGI software.
wsgi.HttpProtocol.log_request = lambda *a: None
# Redirect logging other messages by the underlying WSGI software.
wsgi.HttpProtocol.log_message = \
lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)
eventlet.hubs.use_hub(get_hub())
eventlet.patcher.monkey_patch(all=False, socket=True)
eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
eventlet.debug.hub_exceptions(eventlet_debug)
# utils.LogAdapter stashes name in server; fallback on unadapted loggers
if hasattr(logger, 'server'):
log_name = logger.server
else:
log_name = logger.name
app = loadapp(conf['__file__'], global_conf={'log_name': log_name})
max_clients = int(conf.get('max_clients', '1024'))
pool = RestrictedGreenPool(size=max_clients)
try:
wsgi.server(sock, app, NullLogger(), custom_pool=pool)
except socket.error, err:
if err[0] != errno.EINVAL:
raise
pool.waitall()
# TODO: pull more pieces of this to test more
def run_wsgi(conf_path, app_section, *args, **kwargs):
"""
Runs the server using the specified number of workers.
:param conf_file: Path to paste.deploy style configuration file
:param conf_path: Path to paste.deploy style configuration file/directory
:param app_section: App name from conf file to load config from
"""
# Load configuration, Set logger and Load request processor
try:
(app, conf, logger, log_name) = \
init_request_processor(conf_file, app_section, *args, **kwargs)
init_request_processor(conf_path, app_section, *args, **kwargs)
except ConfigFileError, e:
print e
return
@ -148,34 +245,10 @@ def run_wsgi(conf_file, app_section, *args, **kwargs):
# redirect errors to logger and close stdio
capture_stdio(logger)
def run_server(max_clients):
wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
# Turn off logging requests by the underlying WSGI software.
wsgi.HttpProtocol.log_request = lambda *a: None
# Redirect logging other messages by the underlying WSGI software.
wsgi.HttpProtocol.log_message = \
lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)
eventlet.hubs.use_hub(get_hub())
eventlet.patcher.monkey_patch(all=False, socket=True)
eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
eventlet.debug.hub_exceptions(eventlet_debug)
app = loadapp('config:%s' % conf_file,
global_conf={'log_name': log_name})
pool = RestrictedGreenPool(size=max_clients)
try:
wsgi.server(sock, app, NullLogger(), custom_pool=pool)
except socket.error, err:
if err[0] != errno.EINVAL:
raise
pool.waitall()
max_clients = int(conf.get('max_clients', '1024'))
worker_count = int(conf.get('workers', '1'))
# Useful for profiling [no forks].
if worker_count == 0:
run_server(max_clients)
run_server(conf, logger, sock)
return
def kill_children(*args):
@ -201,7 +274,7 @@ def run_wsgi(conf_file, app_section, *args, **kwargs):
if pid == 0:
signal.signal(signal.SIGHUP, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
run_server(max_clients)
run_server(conf, logger, sock)
logger.notice('Child %d exiting normally' % os.getpid())
return
else:
@ -227,22 +300,22 @@ class ConfigFileError(Exception):
pass
def init_request_processor(conf_file, app_section, *args, **kwargs):
def init_request_processor(conf_path, app_section, *args, **kwargs):
"""
Loads common settings from conf
Sets the logger
Loads the request processor
:param conf_file: Path to paste.deploy style configuration file
:param conf_path: Path to paste.deploy style configuration file/directory
:param app_section: App name from conf file to load config from
:returns: the loaded application entry point
:raises ConfigFileError: Exception is raised for config file error
"""
try:
conf = appconfig('config:%s' % conf_file, name=app_section)
conf = appconfig(conf_path, name=app_section)
except Exception, e:
raise ConfigFileError("Error trying to load config %s: %s" %
(conf_file, e))
raise ConfigFileError("Error trying to load config from %s: %s" %
(conf_path, e))
validate_configuration()
@ -260,7 +333,7 @@ def init_request_processor(conf_file, app_section, *args, **kwargs):
disable_fallocate()
monkey_patch_mimetools()
app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name})
app = loadapp(conf_path, global_conf={'log_name': log_name})
return (app, conf, logger, log_name)

41
test/unit/common/test_manager.py
View File

@ -453,6 +453,47 @@ class TestServer(unittest.TestCase):
conf = self.join_swift_dir(server_name + '.conf')
self.assertEquals(conf_file, conf)
def test_proxy_conf_dir(self):
conf_files = (
'proxy-server.conf.d/00.conf',
'proxy-server.conf.d/01.conf',
)
with temptree(conf_files) as t:
manager.SWIFT_DIR = t
server = manager.Server('proxy')
conf_dirs = server.conf_files()
self.assertEquals(len(conf_dirs), 1)
conf_dir = conf_dirs[0]
proxy_conf_dir = self.join_swift_dir('proxy-server.conf.d')
self.assertEquals(proxy_conf_dir, conf_dir)
def test_conf_dir(self):
conf_files = (
'object-server/object-server.conf-base',
'object-server/1.conf.d/base.conf',
'object-server/1.conf.d/1.conf',
'object-server/2.conf.d/base.conf',
'object-server/2.conf.d/2.conf',
'object-server/3.conf.d/base.conf',
'object-server/3.conf.d/3.conf',
'object-server/4.conf.d/base.conf',
'object-server/4.conf.d/4.conf',
)
with temptree(conf_files) as t:
manager.SWIFT_DIR = t
server = manager.Server('object-replicator')
conf_dirs = server.conf_files()
self.assertEquals(len(conf_dirs), 4)
c1 = self.join_swift_dir('object-server/1.conf.d')
c2 = self.join_swift_dir('object-server/2.conf.d')
c3 = self.join_swift_dir('object-server/3.conf.d')
c4 = self.join_swift_dir('object-server/4.conf.d')
for c in [c1, c2, c3, c4]:
self.assert_(c in conf_dirs)
# test configs returned sorted
sorted_confs = sorted([c1, c2, c3, c4])
self.assertEquals(conf_dirs, sorted_confs)
def test_iter_pid_files(self):
"""
Server.iter_pid_files is kinda boring, test the

101
test/unit/common/test_utils.py
View File

@ -25,6 +25,7 @@ import random
import re
import socket
import sys
from textwrap import dedent
import time
import unittest
from threading import Thread
@ -366,7 +367,7 @@ class TestUtils(unittest.TestCase):
utils.sys.stderr = stde
self.assertRaises(SystemExit, utils.parse_options, once=True,
test_args=[])
self.assert_('missing config file' in stdo.getvalue())
self.assert_('missing config' in stdo.getvalue())
# verify conf file must exist, context manager will delete temp file
with NamedTemporaryFile() as f:
@ -721,6 +722,84 @@ log_name = %(yarr)s'''
os.unlink('/tmp/test')
self.assertRaises(SystemExit, utils.readconf, '/tmp/test')
def test_readconf_dir(self):
config_dir = {
'server.conf.d/01.conf': """
[DEFAULT]
port = 8080
foo = bar
[section1]
name=section1
""",
'server.conf.d/section2.conf': """
[DEFAULT]
port = 8081
bar = baz
[section2]
name=section2
""",
'other-server.conf.d/01.conf': """
[DEFAULT]
port = 8082
[section3]
name=section3
"""
}
# strip indent from test config contents
config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items())
with temptree(*zip(*config_dir.items())) as path:
conf_dir = os.path.join(path, 'server.conf.d')
conf = utils.readconf(conf_dir)
expected = {
'__file__': os.path.join(path, 'server.conf.d'),
'log_name': None,
'section1': {
'port': '8081',
'foo': 'bar',
'bar': 'baz',
'name': 'section1',
},
'section2': {
'port': '8081',
'foo': 'bar',
'bar': 'baz',
'name': 'section2',
},
}
self.assertEquals(conf, expected)
def test_readconf_dir_ignores_hidden_and_nondotconf_files(self):
config_dir = {
'server.conf.d/01.conf': """
[section1]
port = 8080
""",
'server.conf.d/.01.conf.swp': """
[section]
port = 8081
""",
'server.conf.d/01.conf-bak': """
[section]
port = 8082
""",
}
# strip indent from test config contents
config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items())
with temptree(*zip(*config_dir.items())) as path:
conf_dir = os.path.join(path, 'server.conf.d')
conf = utils.readconf(conf_dir)
expected = {
'__file__': os.path.join(path, 'server.conf.d'),
'log_name': None,
'section1': {
'port': '8080',
},
}
self.assertEquals(conf, expected)
def test_drop_privileges(self):
user = getuser()
# over-ride os with mock
@ -925,6 +1004,26 @@ log_name = %(yarr)s'''
for f in [f1, f2, f3, f4]:
self.assert_(f in folder_texts)
def test_search_tree_with_directory_ext_match(self):
files = (
'object-server/object-server.conf-base',
'object-server/1.conf.d/base.conf',
'object-server/1.conf.d/1.conf',
'object-server/2.conf.d/base.conf',
'object-server/2.conf.d/2.conf',
'object-server/3.conf.d/base.conf',
'object-server/3.conf.d/3.conf',
'object-server/4.conf.d/base.conf',
'object-server/4.conf.d/4.conf',
)
with temptree(files) as t:
conf_dirs = utils.search_tree(t, 'object-server', '.conf',
dir_ext='conf.d')
self.assertEquals(len(conf_dirs), 4)
for i in range(4):
conf_dir = os.path.join(t, 'object-server/%d.conf.d' % (i + 1))
self.assert_(conf_dir in conf_dirs)
def test_write_file(self):
with temptree([]) as t:
file_name = os.path.join(t, 'test')

240
test/unit/common/test_wsgi.py
View File

@ -13,24 +13,64 @@
# See the License for the specific language governing permissions and
# limitations under the License.
""" Tests for swift.common.utils """
""" Tests for swift.common.wsgi """
from __future__ import with_statement
import errno
import logging
import mimetools
import socket
import unittest
import os
import pickle
from textwrap import dedent
from gzip import GzipFile
from StringIO import StringIO
from collections import defaultdict
from urllib import quote
from eventlet import listen
import swift
from swift.common.swob import Request
from swift.common import wsgi
from swift.common import wsgi, utils, ring
from test.unit import temptree
from mock import patch
def _fake_rings(tmpdir):
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': 6012},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': 6022}], 30),
GzipFile(os.path.join(tmpdir, 'account.ring.gz'), 'wb'))
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': 6011},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': 6021}], 30),
GzipFile(os.path.join(tmpdir, 'container.ring.gz'), 'wb'))
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': 6010},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': 6020}], 30),
GzipFile(os.path.join(tmpdir, 'object.ring.gz'), 'wb'))
class TestWSGI(unittest.TestCase):
""" Tests for swift.common.wsgi """
def setUp(self):
utils.HASH_PATH_PREFIX = 'startcap'
self._orig_parsetype = mimetools.Message.parsetype
def tearDown(self):
mimetools.Message.parsetype = self._orig_parsetype
def test_monkey_patch_mimetools(self):
sio = StringIO('blah')
self.assertEquals(mimetools.Message(sio).type, 'text/plain')
@ -69,6 +109,90 @@ class TestWSGI(unittest.TestCase):
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEquals(mimetools.Message(sio).subtype, 'html')
def test_init_request_processor(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = catch_errors proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
[filter:catch_errors]
use = egg:swift#catch_errors
"""
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
app, conf, logger, log_name = wsgi.init_request_processor(
conf_file, 'proxy-server')
# verify pipeline is catch_errors -> proxy-servery
expected = swift.common.middleware.catch_errors.CatchErrorMiddleware
self.assert_(isinstance(app, expected))
self.assert_(isinstance(app.app, swift.proxy.server.Application))
# config settings applied to app instance
self.assertEquals(0.2, app.app.conn_timeout)
# appconfig returns values from 'proxy-server' section
expected = {
'__file__': conf_file,
'here': os.path.dirname(conf_file),
'conn_timeout': '0.2',
'swift_dir': t,
}
self.assertEquals(expected, conf)
# logger works
logger.info('testing')
self.assertEquals('proxy-server', log_name)
def test_init_request_processor_from_conf_dir(self):
config_dir = {
'proxy-server.conf.d/pipeline.conf': """
[pipeline:main]
pipeline = catch_errors proxy-server
""",
'proxy-server.conf.d/app.conf': """
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
""",
'proxy-server.conf.d/catch-errors.conf': """
[filter:catch_errors]
use = egg:swift#catch_errors
"""
}
# strip indent from test config contents
config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items())
with temptree(*zip(*config_dir.items())) as conf_root:
conf_dir = os.path.join(conf_root, 'proxy-server.conf.d')
with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f:
f.write('[DEFAULT]\nswift_dir = %s' % conf_root)
_fake_rings(conf_root)
app, conf, logger, log_name = wsgi.init_request_processor(
conf_dir, 'proxy-server')
# verify pipeline is catch_errors -> proxy-servery
expected = swift.common.middleware.catch_errors.CatchErrorMiddleware
self.assert_(isinstance(app, expected))
self.assert_(isinstance(app.app, swift.proxy.server.Application))
# config settings applied to app instance
self.assertEquals(0.2, app.app.conn_timeout)
# appconfig returns values from 'proxy-server' section
expected = {
'__file__': conf_dir,
'here': conf_dir,
'conn_timeout': '0.2',
'swift_dir': conf_root,
}
self.assertEquals(expected, conf)
# logger works
logger.info('testing')
self.assertEquals('proxy-server', log_name)
def test_get_socket(self):
# stubs
conf = {}
@ -170,6 +294,117 @@ class TestWSGI(unittest.TestCase):
wsgi.sleep = old_sleep
wsgi.time = old_time
def test_run_server(self):
config = """
[DEFAULT]
eventlet_debug = yes
client_timeout = 30
swift_dir = TEMPDIR
[pipeline:main]
pipeline = proxy-server
[app:proxy-server]
use = egg:swift#proxy
"""
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
with patch('swift.common.wsgi.wsgi') as _wsgi:
with patch('swift.common.wsgi.eventlet') as _eventlet:
conf = wsgi.appconfig(conf_file)
logger = logging.getLogger('test')
sock = listen(('localhost', 0))
wsgi.run_server(conf, logger, sock)
self.assertEquals('HTTP/1.0',
_wsgi.HttpProtocol.default_request_version)
self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
_eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
_eventlet.patcher.monkey_patch.assert_called_with(all=False,
socket=True)
_eventlet.debug.hub_exceptions.assert_called_with(True)
_wsgi.server.assert_called()
args, kwargs = _wsgi.server.call_args
server_sock, server_app, server_logger = args
self.assertEquals(sock, server_sock)
self.assert_(isinstance(server_app, swift.proxy.server.Application))
self.assert_(isinstance(server_logger, wsgi.NullLogger))
self.assert_('custom_pool' in kwargs)
def test_run_server_conf_dir(self):
config_dir = {
'proxy-server.conf.d/pipeline.conf': """
[pipeline:main]
pipeline = proxy-server
""",
'proxy-server.conf.d/app.conf': """
[app:proxy-server]
use = egg:swift#proxy
""",
'proxy-server.conf.d/default.conf': """
[DEFAULT]
eventlet_debug = yes
client_timeout = 30
"""
}
# strip indent from test config contents
config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items())
with temptree(*zip(*config_dir.items())) as conf_root:
conf_dir = os.path.join(conf_root, 'proxy-server.conf.d')
with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f:
f.write('[DEFAULT]\nswift_dir = %s' % conf_root)
_fake_rings(conf_root)
with patch('swift.common.wsgi.wsgi') as _wsgi:
with patch('swift.common.wsgi.eventlet') as _eventlet:
conf = wsgi.appconfig(conf_dir)
logger = logging.getLogger('test')
sock = listen(('localhost', 0))
wsgi.run_server(conf, logger, sock)
self.assertEquals('HTTP/1.0',
_wsgi.HttpProtocol.default_request_version)
self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
_eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
_eventlet.patcher.monkey_patch.assert_called_with(all=False,
socket=True)
_eventlet.debug.hub_exceptions.assert_called_with(True)
_wsgi.server.assert_called()
args, kwargs = _wsgi.server.call_args
server_sock, server_app, server_logger = args
self.assertEquals(sock, server_sock)
self.assert_(isinstance(server_app, swift.proxy.server.Application))
self.assert_(isinstance(server_logger, wsgi.NullLogger))
self.assert_('custom_pool' in kwargs)
def test_appconfig_dir_ignores_hidden_files(self):
config_dir = {
'server.conf.d/01.conf': """
[app:main]
use = egg:swift#proxy
port = 8080
""",
'server.conf.d/.01.conf.swp': """
[app:main]
use = egg:swift#proxy
port = 8081
""",
}
# strip indent from test config contents
config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items())
with temptree(*zip(*config_dir.items())) as path:
conf_dir = os.path.join(path, 'server.conf.d')
conf = wsgi.appconfig(conf_dir)
expected = {
'__file__': os.path.join(path, 'server.conf.d'),
'here': os.path.join(path, 'server.conf.d'),
'port': '8080',
}
self.assertEquals(conf, expected)
def test_pre_auth_wsgi_input(self):
oldenv = {}
newenv = wsgi.make_pre_authed_env(oldenv)
@ -246,6 +481,7 @@ class TestWSGI(unittest.TestCase):
self.assertEquals(r.body, 'the body')
self.assertEquals(r.environ['swift.source'], 'UT')
class TestWSGIContext(unittest.TestCase):
def test_app_call(self):