openstack/swift
Code Issues Proposed changes

Add targeted config loading to swift-init

Browse Source 
This allows an easier and more explicit way to tell swift-init to run on
specific servers.  For example with an SAIO, this allows you to do
something like:

swift-init object-server.1 reload

to reload just the 1st object server.  A more real world example is when
you are running separate servers for replication.  In this example you
might have an object-server/public.conf and
object-server/replication.conf.  With this change you can do something
like:

swift-init object-server.replication reload

to just reload the replication server.

DocImpact
Change-Id: I5c6046b5ee28e17dadfc5fc53d1d872d9bb8fe48
This commit is contained in:
Chuck Thier 2014-05-08 22:52:15 +00:00
parent 08dba08e76
commit 0a122c1575
6 changed files with 166 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/swift-init
View File

@ -20,7 +20,7 @@ from optparse import OptionParser
from swift.common.manager import Manager, UnknownCommandError, \
KILL_WAIT, RUN_DIR
USAGE = """%prog <server> [<server> ...] <command> [options]
USAGE = """%prog <server>[.config] [<server>[.config] ...] <command> [options]
Commands:
""" + '\n'.join(["%16s: %s" % x for x in Manager.list_commands()])

8
doc/source/admin_guide.rst
View File

@ -1063,6 +1063,14 @@ A graceful shutdown or reload will finish any current requests before
completely stopping the old service. There is also a special case of
`swift-init all <command>`, which will run the command for all swift services.
In cases where there are multiple configs for a service, a specific config
can be managed with ``swift-init <service>.<config> <command>``.
For example, when a separate replication network is used, there might be
`/etc/swift/object-server/public.conf` for the object server and
`/etc/swift/object-server/replication.conf` for the replication services.
In this case, the replication services could be restarted with
``swift-init object-server.replication restart``.
--------------
Object Auditor
--------------

51
swift/common/manager.py
View File

@ -144,23 +144,24 @@ class Manager(object):
"""
def __init__(self, servers, run_dir=RUN_DIR):
server_names = set()
self.server_names = set()
for server in servers:
if server == 'all':
server_names.update(ALL_SERVERS)
self.server_names.update(ALL_SERVERS)
elif server == 'main':
server_names.update(MAIN_SERVERS)
self.server_names.update(MAIN_SERVERS)
elif server == 'rest':
server_names.update(REST_SERVERS)
self.server_names.update(REST_SERVERS)
elif '*' in server:
# convert glob to regex
server_names.update([s for s in ALL_SERVERS if
re.match(server.replace('*', '.*'), s)])
self.server_names.update([
s for s in ALL_SERVERS if
re.match(server.replace('*', '.*'), s)])
else:
server_names.add(server)
self.server_names.add(server)
self.servers = set()
for name in server_names:
for name in self.server_names:
self.servers.add(Server(name, run_dir))
def __iter__(self):
@ -288,8 +289,8 @@ class Manager(object):
"""
kwargs['graceful'] = True
status = 0
for server in self.servers:
m = Manager([server.server])
for server in self.server_names:
m = Manager([server])
status += m.stop(**kwargs)
status += m.start(**kwargs)
return status
@ -345,11 +346,15 @@ class Server(object):
"""
def __init__(self, server, run_dir=RUN_DIR):
if '-' not in server:
server = '%s-server' % server
self.server = server.lower()
self.type = server.rsplit('-', 1)[0]
self.cmd = 'swift-%s' % server
if '.' in self.server:
self.server, self.conf = self.server.rsplit('.', 1)
else:
self.conf = None
if '-' not in self.server:
self.server = '%s-server' % self.server
self.type = self.server.rsplit('-', 1)[0]
self.cmd = 'swift-%s' % self.server
self.procs = []
self.run_dir = run_dir
@ -407,10 +412,15 @@ class Server(object):
:returns: list of conf files
"""
if self.server in STANDALONE_SERVERS:
found_conf_files = search_tree(SWIFT_DIR, self.server + '*',
'.conf', dir_ext='.conf.d')
server_search = self.server
else:
found_conf_files = search_tree(SWIFT_DIR, '%s-server*' % self.type,
server_search = "%s-server" % self.type
if self.conf is not None:
found_conf_files = search_tree(SWIFT_DIR, server_search,
self.conf + '.conf',
dir_ext=self.conf + '.conf.d')
else:
found_conf_files = search_tree(SWIFT_DIR, server_search + '*',
'.conf', dir_ext='.conf.d')
number = kwargs.get('number')
if number:
@ -440,7 +450,12 @@ class Server(object):
:returns: list of pid files
"""
pid_files = search_tree(self.run_dir, '%s*' % self.server)
if self.conf is not None:
pid_files = search_tree(self.run_dir, '%s*' % self.server,
exts=[self.conf + '.pid',
self.conf + '.pid.d'])
else:
pid_files = search_tree(self.run_dir, '%s*' % self.server)
if kwargs.get('number', 0):
conf_files = self.conf_files(**kwargs)
# filter pid_files to match the index of numbered conf_file

5
swift/common/utils.py
View File

@ -1644,7 +1644,7 @@ def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
renamer(tmppath, dest)
def search_tree(root, glob_match, ext='', dir_ext=None):
def search_tree(root, glob_match, ext='', exts=None, dir_ext=None):
"""Look in root, for any files/dirs matching glob, recursively traversing
any found directories looking for files ending with ext
@ -1658,6 +1658,7 @@ def search_tree(root, glob_match, ext='', dir_ext=None):
:returns: list of full paths to matching files, sorted
"""
exts = exts or [ext]
found_files = []
for path in glob.glob(os.path.join(root, glob_match)):
if os.path.isdir(path):
@ -1667,7 +1668,7 @@ def search_tree(root, glob_match, ext='', dir_ext=None):
# the root is a config dir, descend no further
break
for file_ in files:
if ext and not file_.endswith(ext):
if any(exts) and not any(file_.endswith(e) for e in exts):
continue
found_files.append(os.path.join(root, file_))
found_dir = False

93
test/unit/common/test_manager.py
View File

@ -412,6 +412,22 @@ class TestServer(unittest.TestCase):
conf_files = server.conf_files(number=5)
self.assertFalse(conf_files)
# test geting specific conf
conf_files = (
'account-server/1.conf',
'account-server/2.conf',
'account-server/3.conf',
'account-server/4.conf',
)
with temptree(conf_files) as t:
manager.SWIFT_DIR = t
server = manager.Server('account.2')
conf_files = server.conf_files()
self.assertEquals(len(conf_files), 1)
conf_file = conf_files[0]
self.assertEquals(conf_file,
self.join_swift_dir('account-server/2.conf'))
# test verbose & quiet
conf_files = (
'auth-server.ini',
@ -471,6 +487,32 @@ class TestServer(unittest.TestCase):
proxy_conf_dir = self.join_swift_dir('proxy-server.conf.d')
self.assertEquals(proxy_conf_dir, conf_dir)
def test_named_conf_dir(self):
conf_files = (
'object-server/base.conf-template',
'object-server/object-server.conf.d/00_base.conf',
'object-server/object-server.conf.d/10_server.conf',
'object-server/object-replication.conf.d/00_base.conf',
'object-server/object-replication.conf.d/10_server.conf',
)
with temptree(conf_files) as t:
manager.SWIFT_DIR = t
server = manager.Server('object.replication')
conf_dirs = server.conf_files()
self.assertEquals(len(conf_dirs), 1)
conf_dir = conf_dirs[0]
replication_server_conf_dir = self.join_swift_dir(
'object-server/object-replication.conf.d')
self.assertEquals(replication_server_conf_dir, conf_dir)
# and again with no named filter
server = manager.Server('object')
conf_dirs = server.conf_files()
self.assertEquals(len(conf_dirs), 2)
for named_conf in ('server', 'replication'):
conf_dir = self.join_swift_dir(
'object-server/object-%s.conf.d' % named_conf)
self.assert_(conf_dir in conf_dirs)
def test_conf_dir(self):
conf_files = (
'object-server/object-server.conf-base',
@ -498,6 +540,29 @@ class TestServer(unittest.TestCase):
sorted_confs = sorted([c1, c2, c3, c4])
self.assertEquals(conf_dirs, sorted_confs)
def test_named_conf_dir_pid_files(self):
conf_files = (
'object-server/object-server.pid.d',
'object-server/object-replication.pid.d',
)
with temptree(conf_files) as t:
manager.RUN_DIR = t
server = manager.Server('object.replication', run_dir=t)
pid_files = server.pid_files()
self.assertEquals(len(pid_files), 1)
pid_file = pid_files[0]
replication_server_pid = self.join_run_dir(
'object-server/object-replication.pid.d')
self.assertEquals(replication_server_pid, pid_file)
# and again with no named filter
server = manager.Server('object', run_dir=t)
pid_files = server.pid_files()
self.assertEquals(len(pid_files), 2)
for named_pid in ('server', 'replication'):
pid_file = self.join_run_dir(
'object-server/object-%s.pid.d' % named_pid)
self.assert_(pid_file in pid_files)
def test_iter_pid_files(self):
"""
Server.iter_pid_files is kinda boring, test the
@ -581,6 +646,34 @@ class TestServer(unittest.TestCase):
pids = list(server.iter_pid_files(number=5))
self.assertFalse(pids)
# test get pid_files by conf name
conf_files = (
'object-server/1.conf',
'object-server/2.conf',
'object-server/3.conf',
'object-server/4.conf',
)
pid_files = (
('object-server/1.pid', 1),
('object-server/2.pid', 2),
('object-server/5.pid', 5),
)
with temptree(conf_files) as swift_dir:
manager.SWIFT_DIR = swift_dir
files, pids = zip(*pid_files)
with temptree(files, pids) as t:
manager.RUN_DIR = t
server = manager.Server('object.2', run_dir=t)
# test get pid with matching conf
pids = list(server.iter_pid_files())
self.assertEquals(len(pids), 1)
pid_file, pid = pids[0]
self.assertEquals(pid, 2)
pid_two = self.join_run_dir('object-server/2.pid')
self.assertEquals(pid_file, pid_two)
def test_signal_pids(self):
pid_files = (
('proxy-server.pid', 1),

28
test/unit/common/test_utils.py
View File

@ -1242,6 +1242,34 @@ log_name = %(yarr)s'''
conf_dir = os.path.join(t, 'object-server/%d.conf.d' % (i + 1))
self.assert_(conf_dir in conf_dirs)
def test_search_tree_conf_dir_with_named_conf_match(self):
files = (
'proxy-server/proxy-server.conf.d/base.conf',
'proxy-server/proxy-server.conf.d/pipeline.conf',
'proxy-server/proxy-noauth.conf.d/base.conf',
'proxy-server/proxy-noauth.conf.d/pipeline.conf',
)
with temptree(files) as t:
conf_dirs = utils.search_tree(t, 'proxy-server', 'noauth.conf',
dir_ext='noauth.conf.d')
self.assertEquals(len(conf_dirs), 1)
conf_dir = conf_dirs[0]
expected = os.path.join(t, 'proxy-server/proxy-noauth.conf.d')
self.assertEqual(conf_dir, expected)
def test_search_tree_conf_dir_pid_with_named_conf_match(self):
files = (
'proxy-server/proxy-server.pid.d',
'proxy-server/proxy-noauth.pid.d',
)
with temptree(files) as t:
pid_files = utils.search_tree(t, 'proxy-server',
exts=['noauth.pid', 'noauth.pid.d'])
self.assertEquals(len(pid_files), 1)
pid_file = pid_files[0]
expected = os.path.join(t, 'proxy-server/proxy-noauth.pid.d')
self.assertEqual(pid_file, expected)
def test_write_file(self):
with temptree([]) as t:
file_name = os.path.join(t, 'test')