Port to argparse based cfg

Import latest cfg from oslo-incubator with these changes:

 Add deprecated --logdir common opt
 Add deprecated --logfile common opt.
 Allow nova and others to override some logging defaults
 Fixing the trim for ListOp when reading from config file
 Fix set_default() with boolean CLI options
 Improve cfg's argparse sub-parsers support
 Hide the GroupAttr conf and group attributes
 Fix regression with cfg CLI arguments
 Fix broken --help with CommonConfigOpts
 Fix ListOpt to trim whitespace
 updating sphinx documentation
 Don't reference argparse._StoreAction
 Fix minor coding style issue
 Remove ConfigCliParser class
 Add support for positional arguments
 Use stock argparse behaviour for optional args
 Use stock argparse --usage behaviour
 Use stock argparse --version behaviour
 Remove add_option() method
 Completely remove cfg's disable_interspersed_args()
 argparse support for cfg

The main cfg API change is that CONF() no longer returns the un-parsed
CLI arguments. To handle these args, you need to use the support for
positional arguments or sub-parsers.

Switching nova-manage to use sub-parser based CLI arguments means the
following changes in behaviour:

 - no more lazy matching of commands - e.g. 'nova-manage proj q' will
   no longer work. If we find out about common abbreviations used in
   peoples' scripts, we can easily add those.

 - the help output displayed if you run nova-manage without any args
   (or just a category) has changed

 - 'nova-manage version list' is no longer equivalent to
   'nova-manage version'

Change-Id: I19ef3a1c00e97af64d199e27cb1cdc5c63b46a82

bin/nova-clear-rabbit-queues

@@ -48,12 +48,18 @@ from nova.openstack.common import log as logging
 from nova.openstack.common import rpc
 
 
-delete_exchange_opt = cfg.BoolOpt('delete_exchange',
+opts = [
+    cfg.MultiStrOpt('queues',
+                    default=[],
+                    positional=True,
+                    help='Queues to delete'),
+    cfg.BoolOpt('delete_exchange',
                 default=False,
-                                  help='delete nova exchange too.')
+                help='delete nova exchange too.'),
+]
 
 CONF = cfg.CONF
-CONF.register_cli_opt(delete_exchange_opt)
+CONF.register_cli_opts(opts)
 
 
 def delete_exchange(exch):
@@ -69,8 +75,8 @@ def delete_queues(queues):
         x.queue_delete(q)
 
 if __name__ == '__main__':
-    args = config.parse_args(sys.argv)
+    config.parse_args(sys.argv)
     logging.setup("nova")
-    delete_queues(args[1:])
+    delete_queues(CONF.queues)
     if CONF.delete_exchange:
         delete_exchange(CONF.control_exchange)

bin/nova-dhcpbridge

@@ -93,28 +93,44 @@ def init_leases(network_id):
     return network_manager.get_dhcp_leases(ctxt, network_ref)
 
 
+def add_action_parsers(subparsers):
+    parser = subparsers.add_parser('init')
+
+    for action in ['add', 'del', 'old']:
+        parser = subparsers.add_parser(action)
+        parser.add_argument('mac')
+        parser.add_argument('ip')
+        parser.set_defaults(func=globals()[action + '_lease'])
+
+
+CONF.register_cli_opt(
+    cfg.SubCommandOpt('action',
+                      title='Action options',
+                      help='Available dhcpbridge options',
+                      handler=add_action_parsers))
+
+
 def main():
     """Parse environment and arguments and call the approproate action."""
     try:
         config_file = os.environ['CONFIG_FILE']
     except KeyError:
         config_file = os.environ['FLAGFILE']
-    argv = config.parse_args(sys.argv, default_config_files=[config_file])
+
+    config.parse_args(sys.argv, default_config_files=[config_file])
+
     logging.setup("nova")
 
     if int(os.environ.get('TESTING', '0')):
         from nova.tests import fake_flags
 
-    action = argv[1]
+    if CONF.action.name in ['add', 'del', 'old']:
-    if action in ['add', 'del', 'old']:
-        mac = argv[2]
-        ip = argv[3]
         msg = (_("Called '%(action)s' for mac '%(mac)s' with ip '%(ip)s'") %
-               {"action": action,
+               {"action": CONF.action.name,
-                "mac": mac,
+                "mac": CONF.action.mac,
-                "ip": ip})
+                "ip": CONF.action.ip})
         LOG.debug(msg)
-        globals()[action + '_lease'](mac, ip)
+        CONF.action.func(CONF.action.mac, CONF.action.ip)
     else:
         try:
             network_id = int(os.environ.get('NETWORK_ID'))

bin/nova-manage

@@ -56,7 +56,6 @@
 
 import gettext
 import netaddr
-import optparse
 import os
 import sys
 
@@ -106,7 +105,7 @@ QUOTAS = quota.QUOTAS
 # Decorators for actions
 def args(*args, **kwargs):
     def _decorator(func):
-        func.__dict__.setdefault('options', []).insert(0, (args, kwargs))
+        func.__dict__.setdefault('args', []).insert(0, (args, kwargs))
         return func
     return _decorator
 
@@ -764,21 +763,6 @@ class DbCommands(object):
         print migration.db_version()
 
 
-class VersionCommands(object):
-    """Class for exposing the codebase version."""
-
-    def __init__(self):
-        pass
-
-    def list(self):
-        print (_("%(version)s (%(vcs)s)") %
-               {'version': version.version_string(),
-                'vcs': version.version_string_with_vcs()})
-
-    def __call__(self):
-        self.list()
-
-
 class InstanceTypeCommands(object):
     """Class for managing instance types / flavors."""
 
@@ -982,10 +966,10 @@ class GetLogCommands(object):
     def errors(self):
         """Get all of the errors from the log files"""
         error_found = 0
-        if CONF.logdir:
+        if CONF.log_dir:
-            logs = [x for x in os.listdir(CONF.logdir) if x.endswith('.log')]
+            logs = [x for x in os.listdir(CONF.log_dir) if x.endswith('.log')]
             for file in logs:
-                log_file = os.path.join(CONF.logdir, file)
+                log_file = os.path.join(CONF.log_dir, file)
                 lines = [line.strip() for line in open(log_file, "r")]
                 lines.reverse()
                 print_name = 0
@@ -1026,45 +1010,23 @@ class GetLogCommands(object):
             print _('No nova entries in syslog!')
 
 
-CATEGORIES = [
+CATEGORIES = {
-    ('account', AccountCommands),
+    'account': AccountCommands,
-    ('agent', AgentBuildCommands),
+    'agent': AgentBuildCommands,
-    ('db', DbCommands),
+    'db': DbCommands,
-    ('fixed', FixedIpCommands),
+    'fixed': FixedIpCommands,
-    ('flavor', InstanceTypeCommands),
+    'flavor': InstanceTypeCommands,
-    ('floating', FloatingIpCommands),
+    'floating': FloatingIpCommands,
-    ('host', HostCommands),
+    'host': HostCommands,
-    ('instance_type', InstanceTypeCommands),
+    'instance_type': InstanceTypeCommands,
-    ('logs', GetLogCommands),
+    'logs': GetLogCommands,
-    ('network', NetworkCommands),
+    'network': NetworkCommands,
-    ('project', ProjectCommands),
+    'project': ProjectCommands,
-    ('service', ServiceCommands),
+    'service': ServiceCommands,
-    ('shell', ShellCommands),
+    'shell': ShellCommands,
-    ('version', VersionCommands),
+    'vm': VmCommands,
-    ('vm', VmCommands),
+    'vpn': VpnCommands,
-    ('vpn', VpnCommands),
+}
-]
-
-
-def lazy_match(name, key_value_tuples):
-    """Finds all objects that have a key that case insensitively contains
-    [name] key_value_tuples is a list of tuples of the form (key, value)
-    returns a list of tuples of the form (key, value)"""
-    result = []
-    for (k, v) in key_value_tuples:
-        if k.lower().find(name.lower()) == 0:
-            result.append((k, v))
-    if len(result) == 0:
-        print _('%s does not match any options:') % name
-        for k, _v in key_value_tuples:
-            print "\t%s" % k
-        sys.exit(2)
-    if len(result) > 1:
-        print _('%s matched multiple options:') % name
-        for k, _v in result:
-            print "\t%s" % k
-        sys.exit(2)
-    return result
 
 
 def methods_of(obj):
@@ -1077,11 +1039,46 @@ def methods_of(obj):
     return result
 
 
+def add_command_parsers(subparsers):
+    parser = subparsers.add_parser('version')
+
+    parser = subparsers.add_parser('bash-completion')
+    parser.add_argument('query_category', nargs='?')
+
+    for category in CATEGORIES:
+        command_object = CATEGORIES[category]()
+
+        parser = subparsers.add_parser(category)
+        parser.set_defaults(command_object=command_object)
+
+        category_subparsers = parser.add_subparsers(dest='action')
+
+        for (action, action_fn) in methods_of(command_object):
+            parser = category_subparsers.add_parser(action)
+
+            action_kwargs = []
+            for args, kwargs in getattr(action_fn, 'args', []):
+                action_kwargs.append(kwargs['dest'])
+                kwargs['dest'] = 'action_kwarg_' + kwargs['dest']
+                parser.add_argument(*args, **kwargs)
+
+            parser.set_defaults(action_fn=action_fn)
+            parser.set_defaults(action_kwargs=action_kwargs)
+
+            parser.add_argument('action_args', nargs='*')
+
+
+category_opt = cfg.SubCommandOpt('category',
+                                 title='Command categories',
+                                 help='Available categories',
+                                 handler=add_command_parsers)
+
+
 def main():
     """Parse options and call the appropriate class/method."""
-
+    CONF.register_cli_opt(category_opt)
     try:
-        argv = config.parse_args(sys.argv)
+        config.parse_args(sys.argv)
         logging.setup("nova")
     except cfg.ConfigFilesNotFoundError:
         cfgfile = CONF.config_file[-1] if CONF.config_file else None
@@ -1096,69 +1093,33 @@ def main():
         print _('Please re-run nova-manage as root.')
         sys.exit(2)
 
-    script_name = argv.pop(0)
+    if CONF.category.name == "version":
-    if len(argv) < 1:
+        print (_("%(version)s (%(vcs)s)") %
-        print (_("\nOpenStack Nova version: %(version)s (%(vcs)s)\n") %
                {'version': version.version_string(),
                 'vcs': version.version_string_with_vcs()})
-        print script_name + " category action [<args>]"
+        sys.exit(0)
-        print _("Available categories:")
+
-        for k, _v in CATEGORIES:
+    if CONF.category.name == "bash-completion":
-            print "\t%s" % k
+        if not CONF.category.query_category:
-        sys.exit(2)
+            print " ".join(CATEGORIES.keys())
-    category = argv.pop(0)
+        elif CONF.category.query_category in CATEGORIES:
-    if category == "bash-completion":
+            fn = CATEGORIES[CONF.category.query_category]
-        if len(argv) < 1:
-            print " ".join([k for (k, v) in CATEGORIES])
-        else:
-            query_category = argv.pop(0)
-            matches = lazy_match(query_category, CATEGORIES)
-            # instantiate the command group object
-            category, fn = matches[0]
             command_object = fn()
             actions = methods_of(command_object)
             print " ".join([k for (k, v) in actions])
         sys.exit(0)
-    matches = lazy_match(category, CATEGORIES)
-    # instantiate the command group object
-    category, fn = matches[0]
-    command_object = fn()
-    actions = methods_of(command_object)
-    if len(argv) < 1:
-        if hasattr(command_object, '__call__'):
-            action = ''
-            fn = command_object.__call__
-        else:
-            print script_name + " category action [<args>]"
-            print _("Available actions for %s category:") % category
-            for k, _v in actions:
-                print "\t%s" % k
-            sys.exit(2)
-    else:
-        action = argv.pop(0)
-        matches = lazy_match(action, actions)
-        action, fn = matches[0]
 
-    # For not decorated methods
+    fn = CONF.category.action_fn
-    options = getattr(fn, 'options', [])
+    fn_args = [arg.decode('utf-8') for arg in CONF.category.action_args]
-
+    fn_kwargs = {}
-    usage = "%%prog %s %s <args> [options]" % (category, action)
+    for k in CONF.category.action_kwargs:
-    parser = optparse.OptionParser(usage=usage)
+        v = getattr(CONF.category, 'action_kwarg_' + k)
-    for ar, kw in options:
-        parser.add_option(*ar, **kw)
-    (opts, fn_args) = parser.parse_args(argv)
-    fn_kwargs = vars(opts)
-
-    for k, v in fn_kwargs.items():
         if v is None:
-            del fn_kwargs[k]
+            continue
-        elif isinstance(v, basestring):
+        if isinstance(v, basestring):
-            fn_kwargs[k] = v.decode('utf-8')
+            v = v.decode('utf-8')
-        else:
         fn_kwargs[k] = v
 
-    fn_args = [arg.decode('utf-8') for arg in fn_args]
-
     # call the action with the remaining arguments
     try:
         fn(*fn_args, **fn_kwargs)