openstack/glance
Code Issues Proposed changes

Port to argparse based cfg

Browse Source 
Sync the following changes from oslo-incubator:

  479f19c Add deprecated --logdir common opt
  27b2ff4 Add deprecated --logfile common opt.
  9b81289 Allow nova and others to override some logging defaults
  3557d84 Fix ListOpt to trim whitespace
  01ab910 Fix set_default() with boolean CLI options
  af18eaa Improve cfg's argparse sub-parsers support
  f21e1d9 Fix regression with cfg CLI arguments
  ceb4aa7 Fix broken --help with CommonConfigOpts
  5e9503b Hide the GroupAttr conf and group attributes
  b6d24bb updating sphinx documentation
  403509e Don't reference argparse._StoreAction
  e17deb8 Fix minor coding style issue
  0c29e1d Remove ConfigCliParser class
  5b9cb41 Add support for positional arguments
  dbc72a6 Use stock argparse behaviour for optional args
  768a147 Use stock argparse --usage behaviour
  ac180b9 Use stock argparse --version behaviour
  0787e38 Remove add_option() method
  5afead0 Completely remove cfg's disable_interspersed_args()
  5f564b2 argparse support for cfg
  d7b6397 Add a missing comma in a docstring.
  aca1805 cfg: fix required if option has a dash

Changing glance-manage to use argparse sub-parsers does result in an
incompatible change. This no longer works:

  $> glance-manage db_sync --config-file=...

because --config-file is interpreted as an argument to the db_sync
command rather than a global argument. Instead you now have to do:

  $> glance-manage --config-file=... db_sync

The incompatibility is unfortunate, but the new behaviour is pretty
standard.

Change-Id: Ifd43fe6bea4abd7f85ea75e5ae65678b679acd8d
This commit is contained in:
Mark McLoughlin 2012-11-26 11:51:20 +00:00
parent 0e5542ac4c
commit d3490339df
8 changed files with 411 additions and 270 deletions

4
bin/glance-cache-manage

@ -297,11 +297,11 @@ def create_options(parser):
help="Print debugging output") help="Print debugging output")
parser.add_option('-H', '--host', metavar="ADDRESS", default="0.0.0.0", parser.add_option('-H', '--host', metavar="ADDRESS", default="0.0.0.0",
help="Address of Glance API host. " help="Address of Glance API host. "
"Default: %default") "Default: %(default)s")
parser.add_option('-p', '--port', dest="port", metavar="PORT", parser.add_option('-p', '--port', dest="port", metavar="PORT",
type=int, default=9292, type=int, default=9292,
help="Port the Glance API host listens on. " help="Port the Glance API host listens on. "
"Default: %default") "Default: %(default)s")
parser.add_option('-k', '--insecure', dest="insecure", parser.add_option('-k', '--insecure', dest="insecure",
default=False, action="store_true", default=False, action="store_true",
help="Explicitly allow glance to perform \"insecure\" " help="Explicitly allow glance to perform \"insecure\" "

101
bin/glance-control

@ -23,6 +23,7 @@ Thanks for some of the code, Swifties ;)
from __future__ import with_statement from __future__ import with_statement
import argparse
import errno import errno
import fcntl import fcntl
import gettext import gettext
@ -51,12 +52,12 @@ CONF = cfg.CONF
ALL_COMMANDS = ['start', 'status', 'stop', 'shutdown', 'restart', ALL_COMMANDS = ['start', 'status', 'stop', 'shutdown', 'restart',
'reload', 'force-reload'] 'reload', 'force-reload']
ALL_SERVERS = ['glance-api', 'glance-registry', 'glance-scrubber'] ALL_SERVERS = ['api', 'registry', 'scrubber']
GRACEFUL_SHUTDOWN_SERVERS = ['glance-api', 'glance-registry', GRACEFUL_SHUTDOWN_SERVERS = ['glance-api', 'glance-registry',
'glance-scrubber'] 'glance-scrubber']
MAX_DESCRIPTORS = 32768 MAX_DESCRIPTORS = 32768
MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB
USAGE = """%prog [options] <SERVER> <COMMAND> [CONFPATH] USAGE = """%(prog)s [options] <SERVER> <COMMAND> [CONFPATH]
Where <SERVER> is one of: Where <SERVER> is one of:
@ -205,7 +206,7 @@ def do_check_status(pid_file, server):
print "No %s running" % server print "No %s running" % server
def get_pid_file(pid, pid_file): def get_pid_file(server, pid_file):
pid_file = (os.path.abspath(pid_file) if pid_file else pid_file = (os.path.abspath(pid_file) if pid_file else
'/var/run/glance/%s.pid' % server) '/var/run/glance/%s.pid' % server)
dir, file = os.path.split(pid_file) dir, file = os.path.split(pid_file)
@ -259,10 +260,34 @@ def do_stop(server, args, graceful=False):
print 'No %s running' % server print 'No %s running' % server
def add_command_parsers(subparsers):
cmd_parser = argparse.ArgumentParser(add_help=False)
cmd_subparsers = cmd_parser.add_subparsers(dest='command')
for cmd in ALL_COMMANDS:
parser = cmd_subparsers.add_parser(cmd)
parser.add_argument('args', nargs=argparse.REMAINDER)
for server in ALL_SERVERS:
full_name = 'glance-' + server
parser = subparsers.add_parser(server, parents=[cmd_parser])
parser.set_defaults(servers=[full_name])
parser = subparsers.add_parser(full_name, parents=[cmd_parser])
parser.set_defaults(servers=[full_name])
parser = subparsers.add_parser('all', parents=[cmd_parser])
parser.set_defaults(servers=['glance-' + s for s in ALL_SERVERS])
if __name__ == '__main__': if __name__ == '__main__':
exitcode = 0 exitcode = 0
opts = [ opts = [
cfg.SubCommandOpt('server',
title='Server types',
help='Available server types',
handler=add_command_parsers),
cfg.StrOpt('pid-file', cfg.StrOpt('pid-file',
metavar='PATH', metavar='PATH',
help='File to use as pid file. Default: ' help='File to use as pid file. Default: '
@ -283,7 +308,7 @@ if __name__ == '__main__':
] ]
CONF.register_cli_opts(opts) CONF.register_cli_opts(opts)
args = config.parse_args(usage=USAGE) config.parse_args(usage=USAGE)
@gated_by(CONF.await_child) @gated_by(CONF.await_child)
@gated_by(CONF.respawn) @gated_by(CONF.respawn)
@ -293,31 +318,6 @@ if __name__ == '__main__':
mutually_exclusive() mutually_exclusive()
if len(args) < 2:
CONF.print_usage()
sys.exit(1)
server = args.pop(0).lower()
if server == 'all':
servers = ALL_SERVERS
else:
if not server.startswith('glance-'):
server = 'glance-%s' % server
if server not in ALL_SERVERS:
server_list = ", ".join([s.replace('glance-', '')
for s in ALL_SERVERS])
msg = ("Unknown server '%(server)s' specified. Please specify "
"all, or one of the servers: %(server_list)s" % locals())
sys.exit(msg)
servers = [server]
command = args.pop(0).lower()
if command not in ALL_COMMANDS:
command_list = ", ".join(ALL_COMMANDS)
msg = ("Unknown command %(command)s specified. Please specify a "
"command in this list: %(command_list)s" % locals())
sys.exit(msg)
@gated_by(CONF.respawn) @gated_by(CONF.respawn)
def anticipate_respawn(children): def anticipate_respawn(children):
while children: while children:
@ -336,40 +336,41 @@ if __name__ == '__main__':
rsn = 'bouncing' if bouncing else 'deliberately stopped' rsn = 'bouncing' if bouncing else 'deliberately stopped'
print 'Supressed respawn as %s was %s.' % (server, rsn) print 'Supressed respawn as %s was %s.' % (server, rsn)
if command == 'start': if CONF.server.command == 'start':
pid_file = get_pid_file(server, CONF.pid_file)
children = {} children = {}
for server in servers: for server in CONF.server.servers:
args = (pid_file, server, args) pid_file = get_pid_file(server, CONF.pid_file)
args = (pid_file, server, CONF.server.args)
pid = do_start('Start', *args) pid = do_start('Start', *args)
children[pid] = args children[pid] = args
anticipate_respawn(children) anticipate_respawn(children)
if command == 'status': if CONF.server.command == 'status':
for server in servers: for server in CONF.server.servers:
pid_file = get_pid_file(server, CONF.pid_file) pid_file = get_pid_file(server, CONF.pid_file)
do_check_status(pid_file, server) do_check_status(pid_file, server)
if command == 'stop': if CONF.server.command == 'stop':
for server in servers: for server in CONF.server.servers:
do_stop(server, args) do_stop(server, CONF.server.args)
if command == 'shutdown': if CONF.server.command == 'shutdown':
for server in servers: for server in CONF.server.servers:
do_stop(server, args, graceful=True) do_stop(server, CONF.server.args, graceful=True)
if command == 'restart': if CONF.server.command == 'restart':
for server in servers: for server in CONF.server.servers:
do_stop(server, args) do_stop(server, CONF.server.args)
for server in servers: for server in CONF.server.servers:
pid_file = get_pid_file(server, CONF.pid_file) pid_file = get_pid_file(server, CONF.pid_file)
do_start('Restart', pid_file, server, args) do_start('Restart', pid_file, server, CONF.server.args)
if command == 'reload' or command == 'force-reload': if (CONF.server.command == 'reload' or
for server in servers: CONF.server.command == 'force-reload'):
do_stop(server, args, graceful=True) for server in CONF.server.servers:
do_stop(server, CONF.server.args, graceful=True)
pid_file = get_pid_file(server, CONF.pid_file) pid_file = get_pid_file(server, CONF.pid_file)
do_start('Restart', pid_file, server, args) do_start('Restart', pid_file, server, CONF.server.args)
sys.exit(exitcode) sys.exit(exitcode)

78
bin/glance-manage

@ -49,58 +49,65 @@ import glance.db.sqlalchemy.migration
CONF = cfg.CONF CONF = cfg.CONF
def do_db_version(args): def do_db_version():
"""Print database's current migration level""" """Print database's current migration level"""
print glance.db.sqlalchemy.migration.db_version() print glance.db.sqlalchemy.migration.db_version()
def do_upgrade(args): def do_upgrade():
"""Upgrade the database's migration level""" """Upgrade the database's migration level"""
version = args.pop(0) if args else None glance.db.sqlalchemy.migration.upgrade(CONF.command.version)
glance.db.sqlalchemy.migration.upgrade(version)
def do_downgrade(args): def do_downgrade():
"""Downgrade the database's migration level""" """Downgrade the database's migration level"""
if not args: glance.db.sqlalchemy.migration.downgrade(CONF.command.version)
raise exception.MissingArgumentError(
"downgrade requires a version argument")
version = args.pop(0)
glance.db.sqlalchemy.migration.downgrade(version)
def do_version_control(args): def do_version_control():
"""Place a database under migration control""" """Place a database under migration control"""
version = args.pop(0) if args else None glance.db.sqlalchemy.migration.version_control(CONF.command.version)
glance.db.sqlalchemy.migration.version_control(version)
def do_db_sync(args): def do_db_sync():
""" """
Place a database under migration control and upgrade, Place a database under migration control and upgrade,
creating first if necessary. creating first if necessary.
""" """
version = args.pop(0) if args else None glance.db.sqlalchemy.migration.db_sync(CONF.command.version,
current_version = args.pop(0) if args else None CONF.command.current_version)
glance.db.sqlalchemy.migration.db_sync(version, current_version)
def dispatch_cmd(args): def add_command_parsers(subparsers):
"""Search for do_* cmd in this module and then run it""" parser = subparsers.add_parser('db_version')
cmd = args.pop(0) parser.set_defaults(func=do_db_version)
try:
cmd_func = globals()['do_%s' % cmd]
except KeyError:
sys.exit("ERROR: unrecognized command '%s'" % cmd)
try: parser = subparsers.add_parser('upgrade')
cmd_func(args) parser.set_defaults(func=do_upgrade)
except exception.GlanceException, e: parser.add_argument('version', nargs='?')
sys.exit("ERROR: %s" % e)
parser = subparsers.add_parser('downgrade')
parser.set_defaults(func=do_downgrade)
parser.add_argument('version')
parser = subparsers.add_parser('version_control')
parser.set_defaults(func=do_version_control)
parser.add_argument('version', nargs='?')
parser = subparsers.add_parser('db_sync')
parser.set_defaults(func=do_db_sync)
parser.add_argument('version', nargs='?')
parser.add_argument('current_version', nargs='?')
command_opt = cfg.SubCommandOpt('command',
title='Commands',
help='Available commands',
handler=add_command_parsers)
def main(): def main():
CONF.register_cli_opt(command_opt)
try: try:
# We load the glance-registry config section because # We load the glance-registry config section because
# sql_connection is only part of the glance registry. # sql_connection is only part of the glance registry.
@ -109,17 +116,16 @@ def main():
default_cfg_files = cfg.find_config_files(project='glance', default_cfg_files = cfg.find_config_files(project='glance',
prog='glance-registry') prog='glance-registry')
args = config.parse_args(default_config_files=default_cfg_files, config.parse_args(default_config_files=default_cfg_files,
usage="%prog [options] <cmd>") usage="%(prog)s [options] <cmd>")
config.setup_logging() config.setup_logging()
except RuntimeError, e: except RuntimeError, e:
sys.exit("ERROR: %s" % e) sys.exit("ERROR: %s" % e)
if not args: try:
CONF.print_usage() CONF.command.func()
sys.exit(1) except exception.GlanceException, e:
sys.exit("ERROR: %s" % e)
dispatch_cmd(args)
if __name__ == '__main__': if __name__ == '__main__':

12
glance/common/config.py

@ -66,16 +66,16 @@ CONF.register_opts(common_opts)
def parse_args(args=None, usage=None, default_config_files=None): def parse_args(args=None, usage=None, default_config_files=None):
return CONF(args=args, CONF(args=args,
project='glance', project='glance',
version=version.deferred_version_string(prefix="%prog "), version=version.deferred_version_string(prefix="%prog "),
usage=usage, usage=usage,
default_config_files=default_config_files) default_config_files=default_config_files)
def parse_cache_args(args=None): def parse_cache_args(args=None):
config_files = cfg.find_config_files(project='glance', prog='glance-cache') config_files = cfg.find_config_files(project='glance', prog='glance-cache')
return parse_args(args=args, default_config_files=config_files) parse_args(args=args, default_config_files=config_files)
def setup_logging(): def setup_logging():

2
glance/db/__init__.py

@ -28,7 +28,7 @@ sql_connection_opt = cfg.StrOpt('sql_connection',
metavar='CONNECTION', metavar='CONNECTION',
help='A valid SQLAlchemy connection ' help='A valid SQLAlchemy connection '
'string for the registry database. ' 'string for the registry database. '
'Default: %default') 'Default: %(default)s')
CONF = cfg.CONF CONF = cfg.CONF
CONF.register_opt(sql_connection_opt) CONF.register_opt(sql_connection_opt)

464
glance/openstack/common/cfg.py

@ -205,27 +205,11 @@ Option values may reference other values using PEP 292 string substitution::
Note that interpolation can be avoided by using '$$'. Note that interpolation can be avoided by using '$$'.
For command line utilities that dispatch to other command line utilities, the
disable_interspersed_args() method is available. If this this method is called,
then parsing e.g.::
script --verbose cmd --debug /tmp/mything
will no longer return::
['cmd', '/tmp/mything']
as the leftover arguments, but will instead return::
['cmd', '--debug', '/tmp/mything']
i.e. argument parsing is stopped at the first non-option argument.
Options may be declared as required so that an error is raised if the user Options may be declared as required so that an error is raised if the user
does not supply a value for the option. does not supply a value for the option.
Options may be declared as secret so that their values are not leaked into Options may be declared as secret so that their values are not leaked into
log files: log files::
opts = [ opts = [
cfg.StrOpt('s3_store_access_key', secret=True), cfg.StrOpt('s3_store_access_key', secret=True),
@ -234,28 +218,50 @@ log files:
] ]
This module also contains a global instance of the CommonConfigOpts class This module also contains a global instance of the CommonConfigOpts class
in order to support a common usage pattern in OpenStack: in order to support a common usage pattern in OpenStack::
from openstack.common import cfg from glance.openstack.common import cfg
opts = [ opts = [
cfg.StrOpt('bind_host' default='0.0.0.0'), cfg.StrOpt('bind_host', default='0.0.0.0'),
cfg.IntOpt('bind_port', default=9292), cfg.IntOpt('bind_port', default=9292),
] ]
CONF = cfg.CONF CONF = cfg.CONF
CONF.register_opts(opts) CONF.register_opts(opts)
def start(server, app): def start(server, app):
server.start(app, CONF.bind_port, CONF.bind_host) server.start(app, CONF.bind_port, CONF.bind_host)
Positional command line arguments are supported via a 'positional' Opt
constructor argument::
>>> CONF.register_cli_opt(MultiStrOpt('bar', positional=True))
True
>>> CONF(['a', 'b'])
>>> CONF.bar
['a', 'b']
It is also possible to use argparse "sub-parsers" to parse additional
command line arguments using the SubCommandOpt class:
>>> def add_parsers(subparsers):
... list_action = subparsers.add_parser('list')
... list_action.add_argument('id')
...
>>> CONF.register_cli_opt(SubCommandOpt('action', handler=add_parsers))
True
>>> CONF(['list', '10'])
>>> CONF.action.name, CONF.action.id
('list', '10')
""" """
import argparse
import collections import collections
import copy import copy
import functools import functools
import glob import glob
import optparse
import os import os
import string import string
import sys import sys
@ -474,6 +480,13 @@ def _is_opt_registered(opts, opt):
return False return False
def set_defaults(opts, **kwargs):
for opt in opts:
if opt.dest in kwargs:
opt.default = kwargs[opt.dest]
break
class Opt(object): class Opt(object):
"""Base class for all configuration options. """Base class for all configuration options.
@ -489,6 +502,8 @@ class Opt(object):
a single character CLI option name a single character CLI option name
default: default:
the default value of the option the default value of the option
positional:
True if the option is a positional CLI argument
metavar: metavar:
the name shown as the argument to a CLI option in --help output the name shown as the argument to a CLI option in --help output
help: help:
@ -497,8 +512,8 @@ class Opt(object):
multi = False multi = False
def __init__(self, name, dest=None, short=None, default=None, def __init__(self, name, dest=None, short=None, default=None,
metavar=None, help=None, secret=False, required=False, positional=False, metavar=None, help=None,
deprecated_name=None): secret=False, required=False, deprecated_name=None):
"""Construct an Opt object. """Construct an Opt object.
The only required parameter is the option's name. However, it is The only required parameter is the option's name. However, it is
@ -508,6 +523,7 @@ class Opt(object):
:param dest: the name of the corresponding ConfigOpts property :param dest: the name of the corresponding ConfigOpts property
:param short: a single character CLI option name :param short: a single character CLI option name
:param default: the default value of the option :param default: the default value of the option
:param positional: True if the option is a positional CLI argument
:param metavar: the option argument to show in --help :param metavar: the option argument to show in --help
:param help: an explanation of how the option is used :param help: an explanation of how the option is used
:param secret: true iff the value should be obfuscated in log output :param secret: true iff the value should be obfuscated in log output
@ -521,6 +537,7 @@ class Opt(object):
self.dest = dest self.dest = dest
self.short = short self.short = short
self.default = default self.default = default
self.positional = positional
self.metavar = metavar self.metavar = metavar
self.help = help self.help = help
self.secret = secret self.secret = secret
@ -561,64 +578,73 @@ class Opt(object):
:param parser: the CLI option parser :param parser: the CLI option parser
:param group: an optional OptGroup object :param group: an optional OptGroup object
""" """
container = self._get_optparse_container(parser, group) container = self._get_argparse_container(parser, group)
kwargs = self._get_optparse_kwargs(group) kwargs = self._get_argparse_kwargs(group)
prefix = self._get_optparse_prefix('', group) prefix = self._get_argparse_prefix('', group)
self._add_to_optparse(container, self.name, self.short, kwargs, prefix, self._add_to_argparse(container, self.name, self.short, kwargs, prefix,
self.deprecated_name) self.positional, self.deprecated_name)
def _add_to_optparse(self, container, name, short, kwargs, prefix='', def _add_to_argparse(self, container, name, short, kwargs, prefix='',
deprecated_name=None): positional=False, deprecated_name=None):
"""Add an option to an optparse parser or group. """Add an option to an argparse parser or group.
:param container: an optparse.OptionContainer object :param container: an argparse._ArgumentGroup object
:param name: the opt name :param name: the opt name
:param short: the short opt name :param short: the short opt name
:param kwargs: the keyword arguments for add_option() :param kwargs: the keyword arguments for add_argument()
:param prefix: an optional prefix to prepend to the opt name :param prefix: an optional prefix to prepend to the opt name
:param position: whether the optional is a positional CLI argument
:raises: DuplicateOptError if a naming confict is detected :raises: DuplicateOptError if a naming confict is detected
""" """
args = ['--' + prefix + name] def hyphen(arg):
return arg if not positional else ''
args = [hyphen('--') + prefix + name]
if short: if short:
args += ['-' + short] args.append(hyphen('-') + short)
if deprecated_name: if deprecated_name:
args += ['--' + prefix + deprecated_name] args.append(hyphen('--') + prefix + deprecated_name)
for a in args:
if container.has_option(a):
raise DuplicateOptError(a)
container.add_option(*args, **kwargs)
def _get_optparse_container(self, parser, group): try:
"""Returns an optparse.OptionContainer. container.add_argument(*args, **kwargs)
except argparse.ArgumentError as e:
raise DuplicateOptError(e)
:param parser: an optparse.OptionParser def _get_argparse_container(self, parser, group):
"""Returns an argparse._ArgumentGroup.
:param parser: an argparse.ArgumentParser
:param group: an (optional) OptGroup object :param group: an (optional) OptGroup object
:returns: an optparse.OptionGroup if a group is given, else the parser :returns: an argparse._ArgumentGroup if group is given, else parser
""" """
if group is not None: if group is not None:
return group._get_optparse_group(parser) return group._get_argparse_group(parser)
else: else:
return parser return parser
def _get_optparse_kwargs(self, group, **kwargs): def _get_argparse_kwargs(self, group, **kwargs):
"""Build a dict of keyword arguments for optparse's add_option(). """Build a dict of keyword arguments for argparse's add_argument().
Most opt types extend this method to customize the behaviour of the Most opt types extend this method to customize the behaviour of the
options added to optparse. options added to argparse.
:param group: an optional group :param group: an optional group
:param kwargs: optional keyword arguments to add to :param kwargs: optional keyword arguments to add to
:returns: a dict of keyword arguments :returns: a dict of keyword arguments
""" """
dest = self.dest if not self.positional:
if group is not None: dest = self.dest
dest = group.name + '_' + dest if group is not None:
kwargs.update({'dest': dest, dest = group.name + '_' + dest
kwargs['dest'] = dest
else:
kwargs['nargs'] = '?'
kwargs.update({'default': None,
'metavar': self.metavar, 'metavar': self.metavar,
'help': self.help, }) 'help': self.help, })
return kwargs return kwargs
def _get_optparse_prefix(self, prefix, group): def _get_argparse_prefix(self, prefix, group):
"""Build a prefix for the CLI option name, if required. """Build a prefix for the CLI option name, if required.
CLI options in a group are prefixed with the group's name in order CLI options in a group are prefixed with the group's name in order
@ -656,6 +682,11 @@ class BoolOpt(Opt):
_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
'0': False, 'no': False, 'false': False, 'off': False} '0': False, 'no': False, 'false': False, 'off': False}
def __init__(self, *args, **kwargs):
if 'positional' in kwargs:
raise ValueError('positional boolean args not supported')
super(BoolOpt, self).__init__(*args, **kwargs)
def _get_from_config_parser(self, cparser, section): def _get_from_config_parser(self, cparser, section):
"""Retrieve the opt value as a boolean from ConfigParser.""" """Retrieve the opt value as a boolean from ConfigParser."""
def convert_bool(v): def convert_bool(v):
@ -671,21 +702,32 @@ class BoolOpt(Opt):
def _add_to_cli(self, parser, group=None): def _add_to_cli(self, parser, group=None):
"""Extends the base class method to add the --nooptname option.""" """Extends the base class method to add the --nooptname option."""
super(BoolOpt, self)._add_to_cli(parser, group) super(BoolOpt, self)._add_to_cli(parser, group)
self._add_inverse_to_optparse(parser, group) self._add_inverse_to_argparse(parser, group)
def _add_inverse_to_optparse(self, parser, group): def _add_inverse_to_argparse(self, parser, group):
"""Add the --nooptname option to the option parser.""" """Add the --nooptname option to the option parser."""
container = self._get_optparse_container(parser, group) container = self._get_argparse_container(parser, group)
kwargs = self._get_optparse_kwargs(group, action='store_false') kwargs = self._get_argparse_kwargs(group, action='store_false')
prefix = self._get_optparse_prefix('no', group) prefix = self._get_argparse_prefix('no', group)
kwargs["help"] = "The inverse of --" + self.name kwargs["help"] = "The inverse of --" + self.name
self._add_to_optparse(container, self.name, None, kwargs, prefix, self._add_to_argparse(container, self.name, None, kwargs, prefix,
self.deprecated_name) self.positional, self.deprecated_name)
def _get_optparse_kwargs(self, group, action='store_true', **kwargs): def _get_argparse_kwargs(self, group, action='store_true', **kwargs):
"""Extends the base optparse keyword dict for boolean options.""" """Extends the base argparse keyword dict for boolean options."""
return super(BoolOpt,
self)._get_optparse_kwargs(group, action=action, **kwargs) kwargs = super(BoolOpt, self)._get_argparse_kwargs(group, **kwargs)
# metavar has no effect for BoolOpt
if 'metavar' in kwargs:
del kwargs['metavar']
if action != 'store_true':
action = 'store_false'
kwargs['action'] = action
return kwargs
class IntOpt(Opt): class IntOpt(Opt):
@ -697,10 +739,10 @@ class IntOpt(Opt):
return [int(v) for v in self._cparser_get_with_deprecated(cparser, return [int(v) for v in self._cparser_get_with_deprecated(cparser,
section)] section)]
def _get_optparse_kwargs(self, group, **kwargs): def _get_argparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for integer options.""" """Extends the base argparse keyword dict for integer options."""
return super(IntOpt, return super(IntOpt,
self)._get_optparse_kwargs(group, type='int', **kwargs) self)._get_argparse_kwargs(group, type=int, **kwargs)
class FloatOpt(Opt): class FloatOpt(Opt):
@ -712,10 +754,10 @@ class FloatOpt(Opt):
return [float(v) for v in return [float(v) for v in
self._cparser_get_with_deprecated(cparser, section)] self._cparser_get_with_deprecated(cparser, section)]
def _get_optparse_kwargs(self, group, **kwargs): def _get_argparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for float options.""" """Extends the base argparse keyword dict for float options."""
return super(FloatOpt, return super(FloatOpt, self)._get_argparse_kwargs(group,
self)._get_optparse_kwargs(group, type='float', **kwargs) type=float, **kwargs)
class ListOpt(Opt): class ListOpt(Opt):
@ -725,23 +767,26 @@ class ListOpt(Opt):
is a list containing these strings. is a list containing these strings.
""" """
class _StoreListAction(argparse.Action):
"""
An argparse action for parsing an option value into a list.
"""
def __call__(self, parser, namespace, values, option_string=None):
if values is not None:
values = [a.strip() for a in values.split(',')]
setattr(namespace, self.dest, values)
def _get_from_config_parser(self, cparser, section): def _get_from_config_parser(self, cparser, section):
"""Retrieve the opt value as a list from ConfigParser.""" """Retrieve the opt value as a list from ConfigParser."""
return [v.split(',') for v in return [[a.strip() for a in v.split(',')] for v in
self._cparser_get_with_deprecated(cparser, section)] self._cparser_get_with_deprecated(cparser, section)]
def _get_optparse_kwargs(self, group, **kwargs): def _get_argparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for list options.""" """Extends the base argparse keyword dict for list options."""
return super(ListOpt, return Opt._get_argparse_kwargs(self,
self)._get_optparse_kwargs(group, group,
type='string', action=ListOpt._StoreListAction,
action='callback', **kwargs)
callback=self._parse_list,
**kwargs)
def _parse_list(self, option, opt, value, parser):
"""An optparse callback for parsing an option value into a list."""
setattr(parser.values, self.dest, value.split(','))
class MultiStrOpt(Opt): class MultiStrOpt(Opt):
@ -752,10 +797,14 @@ class MultiStrOpt(Opt):
""" """
multi = True multi = True
def _get_optparse_kwargs(self, group, **kwargs): def _get_argparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for multi str options.""" """Extends the base argparse keyword dict for multi str options."""
return super(MultiStrOpt, kwargs = super(MultiStrOpt, self)._get_argparse_kwargs(group)
self)._get_optparse_kwargs(group, action='append') if not self.positional:
kwargs['action'] = 'append'
else:
kwargs['nargs'] = '*'
return kwargs
def _cparser_get_with_deprecated(self, cparser, section): def _cparser_get_with_deprecated(self, cparser, section):
"""If cannot find option as dest try deprecated_name alias.""" """If cannot find option as dest try deprecated_name alias."""
@ -765,6 +814,57 @@ class MultiStrOpt(Opt):
return cparser.get(section, [self.dest], multi=True) return cparser.get(section, [self.dest], multi=True)
class SubCommandOpt(Opt):
"""
Sub-command options allow argparse sub-parsers to be used to parse
additional command line arguments.
The handler argument to the SubCommandOpt contructor is a callable
which is supplied an argparse subparsers object. Use this handler
callable to add sub-parsers.
The opt value is SubCommandAttr object with the name of the chosen
sub-parser stored in the 'name' attribute and the values of other
sub-parser arguments available as additional attributes.
"""
def __init__(self, name, dest=None, handler=None,
title=None, description=None, help=None):
"""Construct an sub-command parsing option.
This behaves similarly to other Opt sub-classes but adds a
'handler' argument. The handler is a callable which is supplied
an subparsers object when invoked. The add_parser() method on
this subparsers object can be used to register parsers for
sub-commands.
:param name: the option's name
:param dest: the name of the corresponding ConfigOpts property
:param title: title of the sub-commands group in help output
:param description: description of the group in help output
:param help: a help string giving an overview of available sub-commands
"""
super(SubCommandOpt, self).__init__(name, dest=dest, help=help)
self.handler = handler
self.title = title
self.description = description
def _add_to_cli(self, parser, group=None):
"""Add argparse sub-parsers and invoke the handler method."""
dest = self.dest
if group is not None:
dest = group.name + '_' + dest
subparsers = parser.add_subparsers(dest=dest,
title=self.title,
description=self.description,
help=self.help)
if not self.handler is None:
self.handler(subparsers)
class OptGroup(object): class OptGroup(object):
""" """
@ -800,19 +900,20 @@ class OptGroup(object):
self.help = help self.help = help
self._opts = {} # dict of dicts of (opt:, override:, default:) self._opts = {} # dict of dicts of (opt:, override:, default:)
self._optparse_group = None self._argparse_group = None
def _register_opt(self, opt): def _register_opt(self, opt, cli=False):
"""Add an opt to this group. """Add an opt to this group.
:param opt: an Opt object :param opt: an Opt object
:param cli: whether this is a CLI option
:returns: False if previously registered, True otherwise :returns: False if previously registered, True otherwise
:raises: DuplicateOptError if a naming conflict is detected :raises: DuplicateOptError if a naming conflict is detected
""" """
if _is_opt_registered(self._opts, opt): if _is_opt_registered(self._opts, opt):
return False return False
self._opts[opt.dest] = {'opt': opt} self._opts[opt.dest] = {'opt': opt, 'cli': cli}
return True return True
@ -824,16 +925,16 @@ class OptGroup(object):
if opt.dest in self._opts: if opt.dest in self._opts:
del self._opts[opt.dest] del self._opts[opt.dest]
def _get_optparse_group(self, parser): def _get_argparse_group(self, parser):
"""Build an optparse.OptionGroup for this group.""" if self._argparse_group is None:
if self._optparse_group is None: """Build an argparse._ArgumentGroup for this group."""
self._optparse_group = optparse.OptionGroup(parser, self.title, self._argparse_group = parser.add_argument_group(self.title,
self.help) self.help)
return self._optparse_group return self._argparse_group
def _clear(self): def _clear(self):
"""Clear this group's option parsing state.""" """Clear this group's option parsing state."""
self._optparse_group = None self._argparse_group = None
class ParseError(iniparser.ParseError): class ParseError(iniparser.ParseError):
@ -928,26 +1029,31 @@ class ConfigOpts(collections.Mapping):
self._groups = {} self._groups = {}
self._args = None self._args = None
self._oparser = None self._oparser = None
self._cparser = None self._cparser = None
self._cli_values = {} self._cli_values = {}
self.__cache = {} self.__cache = {}
self._config_opts = [] self._config_opts = []
self._disable_interspersed_args = False
def _setup(self, project, prog, version, usage, default_config_files): def _pre_setup(self, project, prog, version, usage, default_config_files):
"""Initialize a ConfigOpts object for option parsing.""" """Initialize a ConfigCliParser object for option parsing."""
if prog is None: if prog is None:
prog = os.path.basename(sys.argv[0]) prog = os.path.basename(sys.argv[0])
if default_config_files is None: if default_config_files is None:
default_config_files = find_config_files(project, prog) default_config_files = find_config_files(project, prog)
self._oparser = optparse.OptionParser(prog=prog, self._oparser = argparse.ArgumentParser(prog=prog, usage=usage)
version=version, self._oparser.add_argument('--version',
usage=usage) action='version',
if self._disable_interspersed_args: version=version)
self._oparser.disable_interspersed_args()
return prog, default_config_files
def _setup(self, project, prog, version, usage, default_config_files):
"""Initialize a ConfigOpts object for option parsing."""
self._config_opts = [ self._config_opts = [
MultiStrOpt('config-file', MultiStrOpt('config-file',
@ -1017,18 +1123,23 @@ class ConfigOpts(collections.Mapping):
:raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError, :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError,
RequiredOptError, DuplicateOptError RequiredOptError, DuplicateOptError
""" """
self.clear() self.clear()
prog, default_config_files = self._pre_setup(project,
prog,
version,
usage,
default_config_files)
self._setup(project, prog, version, usage, default_config_files) self._setup(project, prog, version, usage, default_config_files)
self._cli_values, leftovers = self._parse_cli_opts(args) self._cli_values = self._parse_cli_opts(args)
self._parse_config_files() self._parse_config_files()
self._check_required_opts() self._check_required_opts()
return leftovers
def __getattr__(self, name): def __getattr__(self, name):
"""Look up an option value and perform string substitution. """Look up an option value and perform string substitution.
@ -1062,17 +1173,21 @@ class ConfigOpts(collections.Mapping):
@__clear_cache @__clear_cache
def clear(self): def clear(self):
"""Clear the state of the object to before it was called.""" """Clear the state of the object to before it was called.
Any subparsers added using the add_cli_subparsers() will also be
removed as a side-effect of this method.
"""
self._args = None self._args = None
self._cli_values.clear() self._cli_values.clear()
self._oparser = None self._oparser = argparse.ArgumentParser()
self._cparser = None self._cparser = None
self.unregister_opts(self._config_opts) self.unregister_opts(self._config_opts)
for group in self._groups.values(): for group in self._groups.values():
group._clear() group._clear()
@__clear_cache @__clear_cache
def register_opt(self, opt, group=None): def register_opt(self, opt, group=None, cli=False):
"""Register an option schema. """Register an option schema.
Registering an option schema makes any option value which is previously Registering an option schema makes any option value which is previously
@ -1080,17 +1195,19 @@ class ConfigOpts(collections.Mapping):
as an attribute of this object. as an attribute of this object.
:param opt: an instance of an Opt sub-class :param opt: an instance of an Opt sub-class
:param cli: whether this is a CLI option
:param group: an optional OptGroup object or group name :param group: an optional OptGroup object or group name
:return: False if the opt was already register, True otherwise :return: False if the opt was already register, True otherwise
:raises: DuplicateOptError :raises: DuplicateOptError
""" """
if group is not None: if group is not None:
return self._get_group(group, autocreate=True)._register_opt(opt) group = self._get_group(group, autocreate=True)
return group._register_opt(opt, cli)
if _is_opt_registered(self._opts, opt): if _is_opt_registered(self._opts, opt):
return False return False
self._opts[opt.dest] = {'opt': opt} self._opts[opt.dest] = {'opt': opt, 'cli': cli}
return True return True
@ -1116,7 +1233,7 @@ class ConfigOpts(collections.Mapping):
if self._args is not None: if self._args is not None:
raise ArgsAlreadyParsedError("cannot register CLI option") raise ArgsAlreadyParsedError("cannot register CLI option")
return self.register_opt(opt, group, clear_cache=False) return self.register_opt(opt, group, cli=True, clear_cache=False)
@__clear_cache @__clear_cache
def register_cli_opts(self, opts, group=None): def register_cli_opts(self, opts, group=None):
@ -1243,10 +1360,11 @@ class ConfigOpts(collections.Mapping):
for info in group._opts.values(): for info in group._opts.values():
yield info, group yield info, group
def _all_opts(self): def _all_cli_opts(self):
"""A generator function for iteration opts.""" """A generator function for iterating CLI opts."""
for info, group in self._all_opt_infos(): for info, group in self._all_opt_infos():
yield info['opt'], group if info['cli']:
yield info['opt'], group
def _unset_defaults_and_overrides(self): def _unset_defaults_and_overrides(self):
"""Unset any default or override on all options.""" """Unset any default or override on all options."""
@ -1254,31 +1372,6 @@ class ConfigOpts(collections.Mapping):
info.pop('default', None) info.pop('default', None)
info.pop('override', None) info.pop('override', None)
def disable_interspersed_args(self):
"""Set parsing to stop on the first non-option.
If this this method is called, then parsing e.g.
script --verbose cmd --debug /tmp/mything
will no longer return:
['cmd', '/tmp/mything']
as the leftover arguments, but will instead return:
['cmd', '--debug', '/tmp/mything']
i.e. argument parsing is stopped at the first non-option argument.
"""
self._disable_interspersed_args = True
def enable_interspersed_args(self):
"""Set parsing to not stop on the first non-option.
This it the default behaviour."""
self._disable_interspersed_args = False
def find_file(self, name): def find_file(self, name):
"""Locate a file located alongside the config files. """Locate a file located alongside the config files.
@ -1377,6 +1470,9 @@ class ConfigOpts(collections.Mapping):
info = self._get_opt_info(name, group) info = self._get_opt_info(name, group)
opt = info['opt'] opt = info['opt']
if isinstance(opt, SubCommandOpt):
return self.SubCommandAttr(self, group, opt.dest)
if 'override' in info: if 'override' in info:
return info['override'] return info['override']
@ -1401,6 +1497,10 @@ class ConfigOpts(collections.Mapping):
if not opt.multi: if not opt.multi:
return value return value
# argparse ignores default=None for nargs='*'
if opt.positional and not value:
value = opt.default
return value + values return value + values
if values: if values:
@ -1507,7 +1607,7 @@ class ConfigOpts(collections.Mapping):
if ('default' in info or 'override' in info): if ('default' in info or 'override' in info):
continue continue
if self._get(opt.name, group) is None: if self._get(opt.dest, group) is None:
raise RequiredOptError(opt.name, group) raise RequiredOptError(opt.name, group)
def _parse_cli_opts(self, args): def _parse_cli_opts(self, args):
@ -1523,12 +1623,10 @@ class ConfigOpts(collections.Mapping):
""" """
self._args = args self._args = args
for opt, group in self._all_opts(): for opt, group in self._all_cli_opts():
opt._add_to_cli(self._oparser, group) opt._add_to_cli(self._oparser, group)
values, leftovers = self._oparser.parse_args(args) return vars(self._oparser.parse_args(args))
return vars(values), leftovers
class GroupAttr(collections.Mapping): class GroupAttr(collections.Mapping):
@ -1543,12 +1641,12 @@ class ConfigOpts(collections.Mapping):
:param conf: a ConfigOpts object :param conf: a ConfigOpts object
:param group: an OptGroup object :param group: an OptGroup object
""" """
self.conf = conf self._conf = conf
self.group = group self._group = group
def __getattr__(self, name): def __getattr__(self, name):
"""Look up an option value and perform template substitution.""" """Look up an option value and perform template substitution."""
return self.conf._get(name, self.group) return self._conf._get(name, self._group)
def __getitem__(self, key): def __getitem__(self, key):
"""Look up an option value and perform string substitution.""" """Look up an option value and perform string substitution."""
@ -1556,16 +1654,50 @@ class ConfigOpts(collections.Mapping):
def __contains__(self, key): def __contains__(self, key):
"""Return True if key is the name of a registered opt or group.""" """Return True if key is the name of a registered opt or group."""
return key in self.group._opts return key in self._group._opts
def __iter__(self): def __iter__(self):
"""Iterate over all registered opt and group names.""" """Iterate over all registered opt and group names."""
for key in self.group._opts.keys(): for key in self._group._opts.keys():
yield key yield key
def __len__(self): def __len__(self):
"""Return the number of options and option groups.""" """Return the number of options and option groups."""
return len(self.group._opts) return len(self._group._opts)
class SubCommandAttr(object):
"""
A helper class representing the name and arguments of an argparse
sub-parser.
"""
def __init__(self, conf, group, dest):
"""Construct a SubCommandAttr object.
:param conf: a ConfigOpts object
:param group: an OptGroup object
:param dest: the name of the sub-parser
"""
self._conf = conf
self._group = group
self._dest = dest
def __getattr__(self, name):
"""Look up a sub-parser name or argument value."""
if name == 'name':
name = self._dest
if self._group is not None:
name = self._group.name + '_' + name
return self._conf._cli_values[name]
if name in self._conf:
raise DuplicateOptError(name)
try:
return self._conf._cli_values[name]
except KeyError:
raise NoSuchOptError(name)
class StrSubWrapper(object): class StrSubWrapper(object):
@ -1623,19 +1755,21 @@ class CommonConfigOpts(ConfigOpts):
metavar='FORMAT', metavar='FORMAT',
help='A logging.Formatter log message format string which may ' help='A logging.Formatter log message format string which may '
'use any of the available logging.LogRecord attributes. ' 'use any of the available logging.LogRecord attributes. '
'Default: %default'), 'Default: %(default)s'),
StrOpt('log-date-format', StrOpt('log-date-format',
default=DEFAULT_LOG_DATE_FORMAT, default=DEFAULT_LOG_DATE_FORMAT,
metavar='DATE_FORMAT', metavar='DATE_FORMAT',
help='Format string for %(asctime)s in log records. ' help='Format string for %%(asctime)s in log records. '
'Default: %default'), 'Default: %(default)s'),
StrOpt('log-file', StrOpt('log-file',
metavar='PATH', metavar='PATH',
deprecated_name='logfile',
help='(Optional) Name of log file to output to. ' help='(Optional) Name of log file to output to. '
'If not set, logging will go to stdout.'), 'If not set, logging will go to stdout.'),
StrOpt('log-dir', StrOpt('log-dir',
deprecated_name='logdir',
help='(Optional) The directory to keep log files in ' help='(Optional) The directory to keep log files in '
'(will be prepended to --logfile)'), '(will be prepended to --log-file)'),
BoolOpt('use-syslog', BoolOpt('use-syslog',
default=False, default=False,
help='Use syslog for logging.'), help='Use syslog for logging.'),

18
glance/tests/functional/__init__.py

@ -129,9 +129,9 @@ class Server(object):
self.create_database() self.create_database()
cmd = ("%(server_control)s %(server_name)s start " cmd = ("%(server_control)s --pid-file=%(pid_file)s "
"%(conf_file_name)s --pid-file=%(pid_file)s " "%(server_control_options)s "
"%(server_control_options)s" "%(server_name)s start %(conf_file_name)s"
% self.__dict__) % self.__dict__)
return execute(cmd, return execute(cmd,
no_venv=self.no_venv, no_venv=self.no_venv,
@ -147,9 +147,9 @@ class Server(object):
Any kwargs passed to this method will override the configuration Any kwargs passed to this method will override the configuration
value in the conf file used in starting the servers. value in the conf file used in starting the servers.
""" """
cmd = ("%(server_control)s %(server_name)s reload " cmd = ("%(server_control)s --pid-file=%(pid_file)s "
"%(conf_file_name)s --pid-file=%(pid_file)s " "%(server_control_options)s "
"%(server_control_options)s" "%(server_name)s reload %(conf_file_name)s"
% self.__dict__) % self.__dict__)
return execute(cmd, return execute(cmd,
no_venv=self.no_venv, no_venv=self.no_venv,
@ -169,7 +169,7 @@ class Server(object):
conf_file.write('sql_connection = %s' % self.sql_connection) conf_file.write('sql_connection = %s' % self.sql_connection)
conf_file.flush() conf_file.flush()
cmd = ('bin/glance-manage db_sync --config-file %s' cmd = ('bin/glance-manage --config-file %s db_sync'
% conf_filepath) % conf_filepath)
execute(cmd, no_venv=self.no_venv, exec_env=self.exec_env, execute(cmd, no_venv=self.no_venv, exec_env=self.exec_env,
expect_exit=True) expect_exit=True)
@ -178,8 +178,8 @@ class Server(object):
""" """
Spin down the server. Spin down the server.
""" """
cmd = ("%(server_control)s %(server_name)s stop " cmd = ("%(server_control)s --pid-file=%(pid_file)s "
"%(conf_file_name)s --pid-file=%(pid_file)s" "%(server_name)s stop %(conf_file_name)s"
% self.__dict__) % self.__dict__)
return execute(cmd, no_venv=self.no_venv, exec_env=self.exec_env, return execute(cmd, no_venv=self.no_venv, exec_env=self.exec_env,
expect_exit=True) expect_exit=True)

2
glance/tests/functional/test_glance_manage.py

@ -43,7 +43,7 @@ class TestGlanceManage(functional.FunctionalTest):
conf_file.write(self.connection) conf_file.write(self.connection)
conf_file.flush() conf_file.flush()
cmd = ('bin/glance-manage db_sync --config-file %s' % cmd = ('bin/glance-manage --config-file %s db_sync' %
self.conf_filepath) self.conf_filepath)
execute(cmd, raise_error=True) execute(cmd, raise_error=True)