diff --git a/moniker/openstack/common/cfg.py b/moniker/openstack/common/cfg.py index e1cb01382..a35d22c08 100644 --- a/moniker/openstack/common/cfg.py +++ b/moniker/openstack/common/cfg.py @@ -205,8 +205,6 @@ Option values may reference other values using PEP 292 string substitution:: Note that interpolation can be avoided by using '$$'. -FIXME(markmc): document add_cli_subparsers() - Options may be declared as required so that an error is raised if the user does not supply a value for the option. @@ -235,6 +233,28 @@ in order to support a common usage pattern in OpenStack:: def start(server, app): server.start(app, CONF.bind_port, CONF.bind_host) +Positional command line arguments are supported via a 'positional' Opt +constructor argument:: + + >>> CONF.register_cli_opt(MultiStrOpt('bar', positional=True)) + True + >>> CONF(['a', 'b']) + >>> CONF.bar + ['a', 'b'] + +It is also possible to use argparse "sub-parsers" to parse additional +command line arguments using the SubCommandOpt class: + + >>> def add_parsers(subparsers): + ... list_action = subparsers.add_parser('list') + ... list_action.add_argument('id') + ... + >>> CONF.register_cli_opt(SubCommandOpt('action', handler=add_parsers)) + True + >>> CONF(['list', '10']) + >>> CONF.action.name, CONF.action.id + ('list', '10') + """ import argparse @@ -460,6 +480,13 @@ def _is_opt_registered(opts, opt): return False +def set_defaults(opts, **kwargs): + for opt in opts: + if opt.dest in kwargs: + opt.default = kwargs[opt.dest] + break + + class Opt(object): """Base class for all configuration options. @@ -612,7 +639,8 @@ class Opt(object): kwargs['dest'] = dest else: kwargs['nargs'] = '?' - kwargs.update({'metavar': self.metavar, + kwargs.update({'default': None, + 'metavar': self.metavar, 'help': self.help, }) return kwargs @@ -750,7 +778,7 @@ class ListOpt(Opt): def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a list from ConfigParser.""" - return [v.split(',') for v in + return [[a.strip() for a in v.split(',')] for v in self._cparser_get_with_deprecated(cparser, section)] def _get_argparse_kwargs(self, group, **kwargs): @@ -786,6 +814,57 @@ class MultiStrOpt(Opt): return cparser.get(section, [self.dest], multi=True) +class SubCommandOpt(Opt): + + """ + Sub-command options allow argparse sub-parsers to be used to parse + additional command line arguments. + + The handler argument to the SubCommandOpt contructor is a callable + which is supplied an argparse subparsers object. Use this handler + callable to add sub-parsers. + + The opt value is SubCommandAttr object with the name of the chosen + sub-parser stored in the 'name' attribute and the values of other + sub-parser arguments available as additional attributes. + """ + + def __init__(self, name, dest=None, handler=None, + title=None, description=None, help=None): + """Construct an sub-command parsing option. + + This behaves similarly to other Opt sub-classes but adds a + 'handler' argument. The handler is a callable which is supplied + an subparsers object when invoked. The add_parser() method on + this subparsers object can be used to register parsers for + sub-commands. + + :param name: the option's name + :param dest: the name of the corresponding ConfigOpts property + :param title: title of the sub-commands group in help output + :param description: description of the group in help output + :param help: a help string giving an overview of available sub-commands + """ + super(SubCommandOpt, self).__init__(name, dest=dest, help=help) + self.handler = handler + self.title = title + self.description = description + + def _add_to_cli(self, parser, group=None): + """Add argparse sub-parsers and invoke the handler method.""" + dest = self.dest + if group is not None: + dest = group.name + '_' + dest + + subparsers = parser.add_subparsers(dest=dest, + title=self.title, + description=self.description, + help=self.help) + + if not self.handler is None: + self.handler(subparsers) + + class OptGroup(object): """ @@ -951,9 +1030,6 @@ class ConfigOpts(collections.Mapping): self._args = None - # for subparser support, a root parser should be initialized earlier - # and conserved for later use - self._pre_init_parser = None self._oparser = None self._cparser = None self._cli_values = {} @@ -969,18 +1045,7 @@ class ConfigOpts(collections.Mapping): if default_config_files is None: default_config_files = find_config_files(project, prog) - # if _pre_init_parser does not exist, create one - if self._pre_init_parser is None: - self._oparser = argparse.ArgumentParser(prog=prog, usage=usage) - # otherwise, use the pre-initialized parser with subparsers - # and re-initialize parser - else: - self._oparser = self._pre_init_parser - self._oparser.prog = prog - self._oparser.version = version - self._oparser.usage = usage - self._pre_init_parser = None - + self._oparser = argparse.ArgumentParser(prog=prog, usage=usage) self._oparser.add_argument('--version', action='version', version=version) @@ -1101,13 +1166,6 @@ class ConfigOpts(collections.Mapping): """Return the number of options and option groups.""" return len(self._opts) + len(self._groups) - def add_cli_subparsers(self, **kwargs): - # only add subparsers to pre-initialized root parser - # to avoid cleared by self.clear() - if self._pre_init_parser is None: - self._pre_init_parser = argparse.ArgumentParser() - return self._pre_init_parser.add_subparsers(**kwargs) - def reset(self): """Clear the object state and unset overrides and defaults.""" self._unset_defaults_and_overrides() @@ -1115,10 +1173,14 @@ class ConfigOpts(collections.Mapping): @__clear_cache def clear(self): - """Clear the state of the object to before it was called.""" + """Clear the state of the object to before it was called. + + Any subparsers added using the add_cli_subparsers() will also be + removed as a side-effect of this method. + """ self._args = None self._cli_values.clear() - self._oparser = None + self._oparser = argparse.ArgumentParser() self._cparser = None self.unregister_opts(self._config_opts) for group in self._groups.values(): @@ -1408,6 +1470,9 @@ class ConfigOpts(collections.Mapping): info = self._get_opt_info(name, group) opt = info['opt'] + if isinstance(opt, SubCommandOpt): + return self.SubCommandAttr(self, group, opt.dest) + if 'override' in info: return info['override'] @@ -1600,6 +1665,40 @@ class ConfigOpts(collections.Mapping): """Return the number of options and option groups.""" return len(self._group._opts) + class SubCommandAttr(object): + + """ + A helper class representing the name and arguments of an argparse + sub-parser. + """ + + def __init__(self, conf, group, dest): + """Construct a SubCommandAttr object. + + :param conf: a ConfigOpts object + :param group: an OptGroup object + :param dest: the name of the sub-parser + """ + self._conf = conf + self._group = group + self._dest = dest + + def __getattr__(self, name): + """Look up a sub-parser name or argument value.""" + if name == 'name': + name = self._dest + if self._group is not None: + name = self._group.name + '_' + name + return self._conf._cli_values[name] + + if name in self._conf: + raise DuplicateOptError(name) + + try: + return self._conf._cli_values[name] + except KeyError: + raise NoSuchOptError(name) + class StrSubWrapper(object): """ @@ -1664,11 +1763,13 @@ class CommonConfigOpts(ConfigOpts): 'Default: %(default)s'), StrOpt('log-file', metavar='PATH', + deprecated_name='logfile', help='(Optional) Name of log file to output to. ' 'If not set, logging will go to stdout.'), StrOpt('log-dir', + deprecated_name='logdir', help='(Optional) The directory to keep log files in ' - '(will be prepended to --logfile)'), + '(will be prepended to --log-file)'), BoolOpt('use-syslog', default=False, help='Use syslog for logging.'), diff --git a/moniker/openstack/common/log.py b/moniker/openstack/common/log.py index 6f9305271..2ccf0d3b9 100644 --- a/moniker/openstack/common/log.py +++ b/moniker/openstack/common/log.py @@ -289,6 +289,12 @@ def setup(product_name): _setup_logging_from_conf(product_name) +def set_defaults(logging_context_format_string): + cfg.set_defaults(log_opts, + logging_context_format_string= + logging_context_format_string) + + def _find_facility_from_conf(): facility_names = logging.handlers.SysLogHandler.facility_names facility = getattr(logging.handlers.SysLogHandler, diff --git a/moniker/openstack/common/rpc/__init__.py b/moniker/openstack/common/rpc/__init__.py index a9836802d..a1c0a4ab3 100644 --- a/moniker/openstack/common/rpc/__init__.py +++ b/moniker/openstack/common/rpc/__init__.py @@ -50,6 +50,7 @@ rpc_opts = [ default=['moniker.openstack.common.exception', 'nova.exception', 'cinder.exception', + 'exceptions', ], help='Modules of exceptions that are permitted to be recreated' 'upon receiving exception data from an rpc call.'), diff --git a/moniker/openstack/common/rpc/amqp.py b/moniker/openstack/common/rpc/amqp.py index 57f4d8f26..f9299b8f9 100644 --- a/moniker/openstack/common/rpc/amqp.py +++ b/moniker/openstack/common/rpc/amqp.py @@ -153,7 +153,7 @@ class ConnectionContext(rpc_common.Connection): def msg_reply(conf, msg_id, connection_pool, reply=None, failure=None, - ending=False): + ending=False, log_failure=True): """Sends a reply or an error on the channel signified by msg_id. Failure should be a sys.exc_info() tuple. @@ -161,7 +161,8 @@ def msg_reply(conf, msg_id, connection_pool, reply=None, failure=None, """ with ConnectionContext(conf, connection_pool) as conn: if failure: - failure = rpc_common.serialize_remote_exception(failure) + failure = rpc_common.serialize_remote_exception(failure, + log_failure) try: msg = {'result': reply, 'failure': failure} @@ -188,10 +189,10 @@ class RpcContext(rpc_common.CommonRpcContext): return self.__class__(**values) def reply(self, reply=None, failure=None, ending=False, - connection_pool=None): + connection_pool=None, log_failure=True): if self.msg_id: msg_reply(self.conf, self.msg_id, connection_pool, reply, failure, - ending) + ending, log_failure) if ending: self.msg_id = None @@ -285,6 +286,12 @@ class ProxyCallback(object): ctxt.reply(rval, None, connection_pool=self.connection_pool) # This final None tells multicall that it is done. ctxt.reply(ending=True, connection_pool=self.connection_pool) + except rpc_common.ClientException as e: + LOG.debug(_('Expected exception during message handling (%s)') % + e._exc_info[1]) + ctxt.reply(None, e._exc_info, + connection_pool=self.connection_pool, + log_failure=False) except Exception: LOG.exception(_('Exception during message handling')) ctxt.reply(None, sys.exc_info(), diff --git a/moniker/openstack/common/rpc/common.py b/moniker/openstack/common/rpc/common.py index 4a09935bc..1252b8edc 100644 --- a/moniker/openstack/common/rpc/common.py +++ b/moniker/openstack/common/rpc/common.py @@ -18,6 +18,7 @@ # under the License. import copy +import sys import traceback from moniker.openstack.common.gettextutils import _ @@ -210,7 +211,7 @@ def _safe_log(log_func, msg, msg_data): return log_func(msg, msg_data) -def serialize_remote_exception(failure_info): +def serialize_remote_exception(failure_info, log_failure=True): """Prepares exception data to be sent over rpc. Failure_info should be a sys.exc_info() tuple. @@ -218,8 +219,9 @@ def serialize_remote_exception(failure_info): """ tb = traceback.format_exception(*failure_info) failure = failure_info[1] - LOG.error(_("Returning exception %s to caller"), unicode(failure)) - LOG.error(tb) + if log_failure: + LOG.error(_("Returning exception %s to caller"), unicode(failure)) + LOG.error(tb) kwargs = {} if hasattr(failure, 'kwargs'): @@ -324,3 +326,36 @@ class CommonRpcContext(object): context.values['read_deleted'] = read_deleted return context + + +class ClientException(Exception): + """This encapsulates some actual exception that is expected to be + hit by an RPC proxy object. Merely instantiating it records the + current exception information, which will be passed back to the + RPC client without exceptional logging.""" + def __init__(self): + self._exc_info = sys.exc_info() + + +def catch_client_exception(exceptions, func, *args, **kwargs): + try: + return func(*args, **kwargs) + except Exception, e: + if type(e) in exceptions: + raise ClientException() + else: + raise + + +def client_exceptions(*exceptions): + """Decorator for manager methods that raise expected exceptions. + Marking a Manager method with this decorator allows the declaration + of expected exceptions that the RPC layer should not consider fatal, + and not log as if they were generated in a real error scenario. Note + that this will cause listed exceptions to be wrapped in a + ClientException, which is used internally by the RPC layer.""" + def outer(func): + def inner(*args, **kwargs): + return catch_client_exception(exceptions, func, *args, **kwargs) + return inner + return outer diff --git a/moniker/openstack/common/rpc/impl_fake.py b/moniker/openstack/common/rpc/impl_fake.py index f2435ca0f..0a90fd049 100644 --- a/moniker/openstack/common/rpc/impl_fake.py +++ b/moniker/openstack/common/rpc/impl_fake.py @@ -18,11 +18,15 @@ queues. Casts will block, but this is very useful for tests. """ import inspect +# NOTE(russellb): We specifically want to use json, not our own jsonutils. +# jsonutils has some extra logic to automatically convert objects to primitive +# types so that they can be serialized. We want to catch all cases where +# non-primitive types make it into this code and treat it as an error. +import json import time import eventlet -from moniker.openstack.common import jsonutils from moniker.openstack.common.rpc import common as rpc_common CONSUMERS = {} @@ -75,6 +79,8 @@ class Consumer(object): else: res.append(rval) done.send(res) + except rpc_common.ClientException as e: + done.send_exception(e._exc_info[1]) except Exception as e: done.send_exception(e) @@ -124,7 +130,7 @@ def create_connection(conf, new=True): def check_serialize(msg): """Make sure a message intended for rpc can be serialized.""" - jsonutils.dumps(msg) + json.dumps(msg) def multicall(conf, context, topic, msg, timeout=None): @@ -157,6 +163,7 @@ def call(conf, context, topic, msg, timeout=None): def cast(conf, context, topic, msg): + check_serialize(msg) try: call(conf, context, topic, msg) except Exception: diff --git a/moniker/openstack/common/rpc/impl_qpid.py b/moniker/openstack/common/rpc/impl_qpid.py index 3ac888179..9ae276695 100644 --- a/moniker/openstack/common/rpc/impl_qpid.py +++ b/moniker/openstack/common/rpc/impl_qpid.py @@ -279,6 +279,13 @@ class Connection(object): self.consumer_thread = None self.conf = conf + if server_params and 'hostname' in server_params: + # NOTE(russellb) This enables support for cast_to_server. + server_params['qpid_hosts'] = [ + '%s:%d' % (server_params['hostname'], + server_params.get('port', 5672)) + ] + params = { 'qpid_hosts': self.conf.qpid_hosts, 'username': self.conf.qpid_username, diff --git a/moniker/openstack/common/rpc/impl_zmq.py b/moniker/openstack/common/rpc/impl_zmq.py index 887b39a87..b7d5664e4 100644 --- a/moniker/openstack/common/rpc/impl_zmq.py +++ b/moniker/openstack/common/rpc/impl_zmq.py @@ -259,7 +259,14 @@ class InternalContext(object): except greenlet.GreenletExit: # ignore these since they are just from shutdowns pass + except rpc_common.ClientException, e: + LOG.debug(_("Expected exception during message handling (%s)") % + e._exc_info[1]) + return {'exc': + rpc_common.serialize_remote_exception(e._exc_info, + log_failure=False)} except Exception: + LOG.error(_("Exception during message handling")) return {'exc': rpc_common.serialize_remote_exception(sys.exc_info())} diff --git a/moniker/openstack/common/version.py b/moniker/openstack/common/version.py index a19e42265..dae88e37b 100644 --- a/moniker/openstack/common/version.py +++ b/moniker/openstack/common/version.py @@ -24,17 +24,37 @@ import pkg_resources import setup -class _deferred_version_string(object): +class _deferred_version_string(str): """Internal helper class which provides delayed version calculation.""" - def __init__(self, version_info, prefix): - self.version_info = version_info - self.prefix = prefix + + def __new__(cls, version_info, prefix): + new_obj = str.__new__(cls, "") + new_obj._version_info = version_info + new_obj._prefix = prefix + new_obj._cached_version = None + return new_obj + + def _get_cached_version(self): + if not self._cached_version: + self._cached_version = \ + "%s%s" % (self._prefix, + self._version_info.version_string()) + return self._cached_version + + def __len__(self): + return self._get_cached_version().__len__() + + def __contains__(self, item): + return self._get_cached_version().__contains__(item) + + def __getslice__(self, i, j): + return self._get_cached_version().__getslice__(i, j) def __str__(self): - return "%s%s" % (self.prefix, self.version_info.version_string()) + return self._get_cached_version() def __repr__(self): - return "%s%s" % (self.prefix, self.version_info.version_string()) + return self._get_cached_version() class VersionInfo(object): diff --git a/moniker/openstack/common/wsgi.py b/moniker/openstack/common/wsgi.py index 18a8bf838..51b5ceebf 100644 --- a/moniker/openstack/common/wsgi.py +++ b/moniker/openstack/common/wsgi.py @@ -87,7 +87,7 @@ class Service(service.Service): def _run(self, application, socket): """Start a WSGI server in a new green thread.""" - logger = logging.getLogger('eventlet.wsgi.server') + logger = logging.getLogger('eventlet.wsgi') eventlet.wsgi.server(socket, application, custom_pool=self.tg.pool, log=logging.WritableLogger(logger))