From e5a3e09d0263c1a434c0bb76c95e6ddd294b5cae Mon Sep 17 00:00:00 2001 From: Mark McLoughlin Date: Wed, 22 Feb 2012 21:04:11 +0000 Subject: [PATCH 1/3] Update cfg from openstack-common Use openstack-common's update script to sync it to the latest. Add some dire warnings that changes should be made in the upstream copy of the code first. Change-Id: Ib89efdecc7f8a00f2a7fc2824b519f211d3d784c --- HACKING.rst | 15 +++ keystone/openstack/common/README | 13 ++ keystone/openstack/common/cfg.py | 207 ++++++++++++++++++++----------- openstack-common.conf | 7 ++ 4 files changed, 170 insertions(+), 72 deletions(-) create mode 100644 keystone/openstack/common/README create mode 100644 openstack-common.conf diff --git a/HACKING.rst b/HACKING.rst index e6dc568b4c..f703e4e26d 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -191,3 +191,18 @@ without the patch and passes with the patch. For more information on creating unit tests and utilizing the testing infrastructure in OpenStack Nova, please read nova/testing/README.rst. + + +openstack-common +---------------- + +A number of modules from openstack-common are imported into the project. + +These modules are "incubating" in openstack-common and are kept in sync +with the help of openstack-common's update.py script. See: + + http://wiki.openstack.org/CommonLibrary#Incubation + +The copy of the code should never be directly modified here. Please +always update openstack-common first and then run the script to copy +the changes across. diff --git a/keystone/openstack/common/README b/keystone/openstack/common/README new file mode 100644 index 0000000000..def4a172aa --- /dev/null +++ b/keystone/openstack/common/README @@ -0,0 +1,13 @@ +openstack-common +---------------- + +A number of modules from openstack-common are imported into this project. + +These modules are "incubating" in openstack-common and are kept in sync +with the help of openstack-common's update.py script. See: + + http://wiki.openstack.org/CommonLibrary#Incubation + +The copy of the code should never be directly modified here. Please +always update openstack-common first and then run the script to copy +the changes across. diff --git a/keystone/openstack/common/cfg.py b/keystone/openstack/common/cfg.py index cd0a2a0fb7..92eecf3026 100644 --- a/keystone/openstack/common/cfg.py +++ b/keystone/openstack/common/cfg.py @@ -18,7 +18,7 @@ r""" Configuration options which may be set on the command line or in config files. -The schema for each option is defined using the Opt sub-classes e.g.:: +The schema for each option is defined using the Opt sub-classes e.g. common_opts = [ cfg.StrOpt('bind_host', @@ -29,22 +29,20 @@ The schema for each option is defined using the Opt sub-classes e.g.:: help='Port number to listen on') ] -Options can be strings, integers, floats, booleans, lists or 'multi strings':: +Options can be strings, integers, floats, booleans, lists or 'multi strings': - enabled_apis_opt = \ - cfg.ListOpt('enabled_apis', - default=['ec2', 'osapi'], - help='List of APIs to enable by default') + enabled_apis_opt = cfg.ListOpt('enabled_apis', + default=['ec2', 'osapi_compute'], + help='List of APIs to enable by default') DEFAULT_EXTENSIONS = [ 'nova.api.openstack.contrib.standard_extensions' ] - osapi_extension_opt = \ - cfg.MultiStrOpt('osapi_extension', - default=DEFAULT_EXTENSIONS) + osapi_compute_extension_opt = cfg.MultiStrOpt('osapi_compute_extension', + default=DEFAULT_EXTENSIONS) Option schemas are registered with with the config manager at runtime, but -before the option is referenced:: +before the option is referenced: class ExtensionManager(object): @@ -56,11 +54,11 @@ before the option is referenced:: ... def _load_extensions(self): - for ext_factory in self.conf.osapi_extension: + for ext_factory in self.conf.osapi_compute_extension: .... A common usage pattern is for each option schema to be defined in the module or -class which uses the option:: +class which uses the option: opts = ... @@ -75,7 +73,7 @@ class which uses the option:: An option may optionally be made available via the command line. Such options must registered with the config manager before the command line is parsed (for -the purposes of --help and CLI arg validation):: +the purposes of --help and CLI arg validation): cli_opts = [ cfg.BoolOpt('verbose', @@ -91,13 +89,12 @@ the purposes of --help and CLI arg validation):: def add_common_opts(conf): conf.register_cli_opts(cli_opts) -The config manager has a single CLI option defined by default, --config-file:: +The config manager has a single CLI option defined by default, --config-file: class ConfigOpts(object): - config_file_opt = \ - MultiStrOpt('config-file', - ... + config_file_opt = MultiStrOpt('config-file', + ... def __init__(self, ...): ... @@ -105,7 +102,7 @@ The config manager has a single CLI option defined by default, --config-file:: Option values are parsed from any supplied config files using SafeConfigParser. If none are specified, a default set is used e.g. glance-api.conf and -glance-common.conf:: +glance-common.conf: glance-api.conf: [DEFAULT] @@ -120,7 +117,7 @@ are parsed in order, with values in later files overriding those in earlier files. The parsing of CLI args and config files is initiated by invoking the config -manager e.g.:: +manager e.g. conf = ConfigOpts() conf.register_opt(BoolOpt('verbose', ...)) @@ -128,34 +125,26 @@ manager e.g.:: if conf.verbose: ... -Options can be registered as belonging to a group:: +Options can be registered as belonging to a group: rabbit_group = cfg.OptionGroup(name='rabbit', title='RabbitMQ options') - rabbit_host_opt = \ - cfg.StrOpt('host', - group='rabbit', - default='localhost', - help='IP/hostname to listen on'), - rabbit_port_opt = \ - cfg.IntOpt('port', - default=5672, - help='Port number to listen on') - rabbit_ssl_opt = \ - conf.BoolOpt('use_ssl', - default=False, - help='Whether to support SSL connections') + rabbit_host_opt = cfg.StrOpt('host', + default='localhost', + help='IP/hostname to listen on'), + rabbit_port_opt = cfg.IntOpt('port', + default=5672, + help='Port number to listen on') def register_rabbit_opts(conf): conf.register_group(rabbit_group) - # options can be registered under a group in any of these ways: - conf.register_opt(rabbit_host_opt) + # options can be registered under a group in either of these ways: + conf.register_opt(rabbit_host_opt, group=rabbit_group) conf.register_opt(rabbit_port_opt, group='rabbit') - conf.register_opt(rabbit_ssl_opt, group=rabbit_group) If no group is specified, options belong to the 'DEFAULT' section of config -files:: +files: glance-api.conf: [DEFAULT] @@ -172,11 +161,11 @@ files:: Command-line options in a group are automatically prefixed with the group name: - --rabbit-host localhost --rabbit-use-ssl False + --rabbit-host localhost --rabbit-port 9999 Option values in the default group are referenced as attributes/properties on the config manager; groups are also attributes on the config manager, with -attributes for each of the options associated with the group:: +attributes for each of the options associated with the group: server.start(app, conf.bind_port, conf.bind_host, conf) @@ -185,7 +174,7 @@ attributes for each of the options associated with the group:: port=conf.rabbit.port, ...) -Option values may reference other values using PEP 292 string substitution:: +Option values may reference other values using PEP 292 string substitution: opts = [ cfg.StrOpt('state_path', @@ -200,14 +189,31 @@ Option values may reference other values using PEP 292 string substitution:: ] Note that interpolation can be avoided by using '$$'. + +For command line utilities that dispatch to other command line utilities, the +disable_interspersed_args() method is available. If this this method is called, +then parsing e.g. + + script --verbose cmd --debug /tmp/mything + +will no longer return: + + ['cmd', '/tmp/mything'] + +as the leftover arguments, but will instead return: + + ['cmd', '--debug', '/tmp/mything'] + +i.e. argument parsing is stopped at the first non-option argument. """ -import sys +import collections import ConfigParser import copy import optparse import os import string +import sys class Error(Exception): @@ -224,13 +230,13 @@ class ArgsAlreadyParsedError(Error): """Raised if a CLI opt is registered after parsing.""" def __str__(self): - ret = 'arguments already parsed' + ret = "arguments already parsed" if self.msg: - ret += ': ' + self.msg + ret += ": " + self.msg return ret -class NoSuchOptError(Error): +class NoSuchOptError(Error, AttributeError): """Raised if an opt which doesn't exist is referenced.""" def __init__(self, opt_name, group=None): @@ -239,9 +245,9 @@ class NoSuchOptError(Error): def __str__(self): if self.group is None: - return 'no such option: %s' % self.opt_name + return "no such option: %s" % self.opt_name else: - return 'no such option in group %s: %s' % (self.group.name, + return "no such option in group %s: %s" % (self.group.name, self.opt_name) @@ -252,7 +258,7 @@ class NoSuchGroupError(Error): self.group_name = group_name def __str__(self): - return 'no such group: %s' % self.group_name + return "no such group: %s" % self.group_name class DuplicateOptError(Error): @@ -262,14 +268,14 @@ class DuplicateOptError(Error): self.opt_name = opt_name def __str__(self): - return 'duplicate option: %s' % self.opt_name + return "duplicate option: %s" % self.opt_name class TemplateSubstitutionError(Error): """Raised if an error occurs substituting a variable in an opt value.""" def __str__(self): - return 'template substitution error: %s' % self.msg + return "template substitution error: %s" % self.msg class ConfigFilesNotFoundError(Error): @@ -279,8 +285,8 @@ class ConfigFilesNotFoundError(Error): self.config_files = config_files def __str__(self): - return 'Failed to read some config files: %s' % \ - string.join(self.config_files, ',') + return ('Failed to read some config files: %s' % + string.join(self.config_files, ',')) class ConfigFileParseError(Error): @@ -425,7 +431,7 @@ class Opt(object): :param cparser: a ConfigParser object :param section: a section name """ - return cparser.get(section, self.dest) + return cparser.get(section, self.dest, raw=True) def _add_to_cli(self, parser, group=None): """Makes the option available in the command line interface. @@ -541,6 +547,7 @@ class BoolOpt(Opt): container = self._get_optparse_container(parser, group) kwargs = self._get_optparse_kwargs(group, action='store_false') prefix = self._get_optparse_prefix('no', group) + kwargs["help"] = "The inverse of --" + self.name self._add_to_optparse(container, self.name, None, kwargs, prefix) def _get_optparse_kwargs(self, group, action='store_true', **kwargs): @@ -613,8 +620,8 @@ class MultiStrOpt(Opt): """Retrieve the opt value as a multistr from ConfigParser.""" # FIXME(markmc): values spread across the CLI and multiple # config files should be appended - value = \ - super(MultiStrOpt, self)._get_from_config_parser(cparser, section) + value = super(MultiStrOpt, self)._get_from_config_parser(cparser, + section) return value if value is None else [value] def _get_optparse_kwargs(self, group, **kwargs): @@ -657,7 +664,7 @@ class OptGroup(object): self.title = title self.help = help - self._opts = {} # dict of dicts of {opt:, override:, default:) + self._opts = {} # dict of dicts of (opt:, override:, default:) self._optparse_group = None def _register_opt(self, opt): @@ -677,12 +684,12 @@ class OptGroup(object): def _get_optparse_group(self, parser): """Build an optparse.OptionGroup for this group.""" if self._optparse_group is None: - self._optparse_group = \ - optparse.OptionGroup(parser, self.title, self.help) + self._optparse_group = optparse.OptionGroup(parser, self.title, + self.help) return self._optparse_group -class ConfigOpts(object): +class ConfigOpts(collections.Mapping): """ Config options which may be set on the command line or in config files. @@ -732,7 +739,7 @@ class ConfigOpts(object): usage=self.usage) self._cparser = None - self.register_cli_opt(\ + self.register_cli_opt( MultiStrOpt('config-file', default=self.default_config_files, metavar='PATH', @@ -777,6 +784,23 @@ class ConfigOpts(object): """ return self._substitute(self._get(name)) + def __getitem__(self, key): + """Look up an option value and perform string substitution.""" + return self.__getattr__(key) + + def __contains__(self, key): + """Return True if key is the name of a registered opt or group.""" + return key in self._opts or key in self._groups + + def __iter__(self): + """Iterate over all registered opt and group names.""" + for key in self._opts.keys() + self._groups.keys(): + yield key + + def __len__(self): + """Return the number of options and option groups.""" + return len(self._opts) + len(self._groups) + def reset(self): """Reset the state of the object to before it was called.""" self._args = None @@ -881,6 +905,31 @@ class ConfigOpts(object): opt_info = self._get_opt_info(name, group) opt_info['default'] = default + def disable_interspersed_args(self): + """Set parsing to stop on the first non-option. + + If this this method is called, then parsing e.g. + + script --verbose cmd --debug /tmp/mything + + will no longer return: + + ['cmd', '/tmp/mything'] + + as the leftover arguments, but will instead return: + + ['cmd', '--debug', '/tmp/mything'] + + i.e. argument parsing is stopped at the first non-option argument. + """ + self._oparser.disable_interspersed_args() + + def enable_interspersed_args(self): + """Set parsing to not stop on the first non-option. + + This it the default behaviour.""" + self._oparser.enable_interspersed_args() + def log_opt_values(self, logger, lvl): """Log the value of all registered opts. @@ -901,7 +950,7 @@ class ConfigOpts(object): logger.log(lvl, "%-30s = %s", opt_name, getattr(self, opt_name)) for group_name in self._groups: - group_attr = self.GroupAttr(self, group_name) + group_attr = self.GroupAttr(self, self._get_group(group_name)) for opt_name in sorted(self._groups[group_name]._opts): logger.log(lvl, "%-30s = %s", "%s.%s" % (group_name, opt_name), @@ -917,16 +966,13 @@ class ConfigOpts(object): """Look up an option value. :param name: the opt name (or 'dest', more precisely) - :param group: an option OptGroup + :param group: an OptGroup :returns: the option value, or a GroupAttr object :raises: NoSuchOptError, NoSuchGroupError, ConfigFileValueError, TemplateSubstitutionError """ if group is None and name in self._groups: - return self.GroupAttr(self, name) - - if group is not None: - group = self._get_group(group) + return self.GroupAttr(self, self._get_group(name)) info = self._get_opt_info(name, group) default, opt, override = map(lambda k: info[k], sorted(info.keys())) @@ -1028,17 +1074,18 @@ class ConfigOpts(object): not_read_ok = filter(lambda f: f not in read_ok, config_files) raise ConfigFilesNotFoundError(not_read_ok) - class GroupAttr(object): + class GroupAttr(collections.Mapping): """ - A helper class representing the option values of a group as attributes. + A helper class representing the option values of a group as a mapping + and attributes. """ def __init__(self, conf, group): """Construct a GroupAttr object. :param conf: a ConfigOpts object - :param group: a group name or OptGroup object + :param group: an OptGroup object """ self.conf = conf self.group = group @@ -1047,6 +1094,23 @@ class ConfigOpts(object): """Look up an option value and perform template substitution.""" return self.conf._substitute(self.conf._get(name, self.group)) + def __getitem__(self, key): + """Look up an option value and perform string substitution.""" + return self.__getattr__(key) + + def __contains__(self, key): + """Return True if key is the name of a registered opt or group.""" + return key in self.group._opts + + def __iter__(self): + """Iterate over all registered opt and group names.""" + for key in self.group._opts.keys(): + yield key + + def __len__(self): + """Return the number of options and option groups.""" + return len(self.group._opts) + class StrSubWrapper(object): """ @@ -1116,13 +1180,12 @@ class CommonConfigOpts(ConfigOpts): StrOpt('log-dir', help='(Optional) The directory to keep log files in ' '(will be prepended to --logfile)'), - StrOpt('syslog-log-facility', - default='LOG_USER', - help='(Optional) The syslog facility to use when logging ' - 'to syslog (defaults to LOG_USER)'), BoolOpt('use-syslog', default=False, help='Use syslog for logging.'), + StrOpt('syslog-log-facility', + default='LOG_USER', + help='syslog facility to receive log lines') ] def __init__(self, **kwargs): diff --git a/openstack-common.conf b/openstack-common.conf new file mode 100644 index 0000000000..db2171c1e0 --- /dev/null +++ b/openstack-common.conf @@ -0,0 +1,7 @@ +[DEFAULT] + +# The list of modules to copy from openstack-common +modules=cfg + +# The base module to hold the copy of openstack.common +base=keystone From 762b461720fced80435de818f9f843ced5059035 Mon Sep 17 00:00:00 2001 From: Mark McLoughlin Date: Wed, 22 Feb 2012 21:04:11 +0000 Subject: [PATCH 2/3] Remove cfg dict mixin cfg now implements the collections.Mapping ABC Use set_override() in tests rather than directly setting the attribute since the cfg iterator doesn't iterate the attributes. Change-Id: I9db09c778b0fed5e19e836ce4e6006f790e2fca0 --- keystone/config.py | 10 ---------- keystone/test.py | 17 ++++++++++++++--- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/keystone/config.py b/keystone/config.py index 3b1bcab2d2..ab2f429c0a 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -32,16 +32,6 @@ class ConfigMixin(object): kw.setdefault('args', []) return super(ConfigMixin, self).__call__(*args, **kw) - def __getitem__(self, key, default=None): - return getattr(self, key, default) - - def __setitem__(self, key, value): - return setattr(self, key, value) - - def iteritems(self): - for k in self._opts: - yield (k, getattr(self, k)) - def print_help(self): self._oparser.print_help() diff --git a/keystone/test.py b/keystone/test.py index 22ed3fabc3..b830134060 100644 --- a/keystone/test.py +++ b/keystone/test.py @@ -118,6 +118,7 @@ class TestCase(unittest.TestCase): super(TestCase, self).__init__(*args, **kw) self._paths = [] self._memo = {} + self._overrides = [] def setUp(self): super(TestCase, self).setUp() @@ -132,9 +133,20 @@ class TestCase(unittest.TestCase): if path in sys.path: sys.path.remove(path) kvs.INMEMDB.clear() - CONF.reset() + self.reset_opts() super(TestCase, self).tearDown() + def opt(self, **kw): + for k, v in kw.iteritems(): + CONF.set_override(k, v) + self._overrides.append(k) + + def reset_opts(self): + for k in self._overrides: + CONF.set_override(k, None) + self._overrides = [] + CONF.reset() + def load_backends(self): """Hacky shortcut to load the backends for data manipulation.""" self.identity_api = utils.import_object(CONF.identity.driver) @@ -201,8 +213,7 @@ class TestCase(unittest.TestCase): # Service catalog tests need to know the port we ran on. port = server.socket_info['socket'][1] - CONF.public_port = port - CONF.admin_port = port + self.opt(public_port=port, admin_port=port) return server def client(self, app, *args, **kw): From 1f119bc91f178df43f1cec6df3247f6f6a4e0c36 Mon Sep 17 00:00:00 2001 From: Mark McLoughlin Date: Wed, 22 Feb 2012 21:04:11 +0000 Subject: [PATCH 3/3] Use cfg's new print_help() method Change-Id: Ib8e5dc50bc9773d13b8d9c741676477aa0f001e0 --- keystone/config.py | 3 --- keystone/openstack/common/cfg.py | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/keystone/config.py b/keystone/config.py index ab2f429c0a..18962b292b 100644 --- a/keystone/config.py +++ b/keystone/config.py @@ -32,9 +32,6 @@ class ConfigMixin(object): kw.setdefault('args', []) return super(ConfigMixin, self).__call__(*args, **kw) - def print_help(self): - self._oparser.print_help() - def set_usage(self, usage): self.usage = usage self._oparser.usage = usage diff --git a/keystone/openstack/common/cfg.py b/keystone/openstack/common/cfg.py index 92eecf3026..e92a574acf 100644 --- a/keystone/openstack/common/cfg.py +++ b/keystone/openstack/common/cfg.py @@ -962,6 +962,10 @@ class ConfigOpts(collections.Mapping): """Print the usage message for the current program.""" self._oparser.print_usage(file) + def print_help(self, file=None): + """Print the help message for the current program.""" + self._oparser.print_help(file) + def _get(self, name, group=None): """Look up an option value.