diff --git a/heat/context.py b/heat/context.py index 4ab7afe29c..914b494076 100644 --- a/heat/context.py +++ b/heat/context.py @@ -22,7 +22,7 @@ import copy import logging -from heat import local +from heat.openstack.common import local from heat.common import utils diff --git a/heat/openstack/common/cfg.py b/heat/openstack/common/cfg.py index 1005b2f788..fa15a22236 100644 --- a/heat/openstack/common/cfg.py +++ b/heat/openstack/common/cfg.py @@ -101,9 +101,9 @@ The config manager has a single CLI option defined by default, --config-file:: ... self.register_cli_opt(self.config_file_opt) -Option values are parsed from any supplied config files using SafeConfigParser. -If none are specified, a default set is used e.g. glance-api.conf and -glance-common.conf:: +Option values are parsed from any supplied config files using +openstack.common.iniparser. If none are specified, a default set is used +e.g. glance-api.conf and glance-common.conf:: glance-api.conf: [DEFAULT] @@ -128,8 +128,8 @@ manager e.g.:: Options can be registered as belonging to a group:: - rabbit_group = cfg.OptionGroup(name='rabbit', - title='RabbitMQ options') + rabbit_group = cfg.OptGroup(name='rabbit', + title='RabbitMQ options') rabbit_host_opt = cfg.StrOpt('host', default='localhost', @@ -207,16 +207,27 @@ as the leftover arguments, but will instead return:: ['cmd', '--debug', '/tmp/mything'] i.e. argument parsing is stopped at the first non-option argument. + +Options may be declared as secret so that their values are not leaked into +log files: + + opts = [ + cfg.StrOpt('s3_store_access_key', secret=True), + cfg.StrOpt('s3_store_secret_key', secret=True), + ... + ] + """ import collections -import ConfigParser import copy import optparse import os import string import sys +from heat.openstack.common import iniparser + class Error(Exception): """Base class for cfg exceptions.""" @@ -398,9 +409,10 @@ class Opt(object): help: an string explaining how the options value is used """ + multi = False - def __init__(self, name, dest=None, short=None, - default=None, metavar=None, help=None): + def __init__(self, name, dest=None, short=None, default=None, + metavar=None, help=None, secret=False): """Construct an Opt object. The only required parameter is the option's name. However, it is @@ -412,6 +424,7 @@ class Opt(object): :param default: the default value of the option :param metavar: the option argument to show in --help :param help: an explanation of how the option is used + :param secret: true iff the value should be obfuscated in log output """ self.name = name if dest is None: @@ -422,9 +435,10 @@ class Opt(object): self.default = default self.metavar = metavar self.help = help + self.secret = secret def _get_from_config_parser(self, cparser, section): - """Retrieves the option value from a ConfigParser object. + """Retrieves the option value from a MultiConfigParser object. This is the method ConfigOpts uses to look up the option value from config files. Most opt types override this method in order to perform @@ -433,7 +447,7 @@ class Opt(object): :param cparser: a ConfigParser object :param section: a section name """ - return cparser.get(section, self.dest, raw=True) + return cparser.get(section, self.dest) def _add_to_cli(self, parser, group=None): """Makes the option available in the command line interface. @@ -535,9 +549,19 @@ class BoolOpt(Opt): 1/0, yes/no, true/false or on/off. """ + _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, + '0': False, 'no': False, 'false': False, 'off': False} + def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a boolean from ConfigParser.""" - return cparser.getboolean(section, self.dest) + def convert_bool(v): + value = self._boolean_states.get(v.lower()) + if value is None: + raise ValueError('Unexpected boolean value %r' % v) + + return value + + return [convert_bool(v) for v in cparser.get(section, self.dest)] def _add_to_cli(self, parser, group=None): """Extends the base class method to add the --nooptname option.""" @@ -564,7 +588,7 @@ class IntOpt(Opt): def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a integer from ConfigParser.""" - return cparser.getint(section, self.dest) + return [int(v) for v in cparser.get(section, self.dest)] def _get_optparse_kwargs(self, group, **kwargs): """Extends the base optparse keyword dict for integer options.""" @@ -578,7 +602,7 @@ class FloatOpt(Opt): def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a float from ConfigParser.""" - return cparser.getfloat(section, self.dest) + return [float(v) for v in cparser.get(section, self.dest)] def _get_optparse_kwargs(self, group, **kwargs): """Extends the base optparse keyword dict for float options.""" @@ -595,7 +619,7 @@ class ListOpt(Opt): def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a list from ConfigParser.""" - return cparser.get(section, self.dest).split(',') + return [v.split(',') for v in cparser.get(section, self.dest)] def _get_optparse_kwargs(self, group, **kwargs): """Extends the base optparse keyword dict for list options.""" @@ -617,14 +641,7 @@ class MultiStrOpt(Opt): Multistr opt values are string opts which may be specified multiple times. The opt value is a list containing all the string values specified. """ - - def _get_from_config_parser(self, cparser, section): - """Retrieve the opt value as a multistr from ConfigParser.""" - # FIXME(markmc): values spread across the CLI and multiple - # config files should be appended - value = super(MultiStrOpt, self)._get_from_config_parser(cparser, - section) - return value if value is None else [value] + multi = True def _get_optparse_kwargs(self, group, **kwargs): """Extends the base optparse keyword dict for multi str options.""" @@ -691,6 +708,69 @@ class OptGroup(object): return self._optparse_group +class ParseError(iniparser.ParseError): + def __init__(self, msg, lineno, line, filename): + super(ParseError, self).__init__(msg, lineno, line) + self.filename = filename + + def __str__(self): + return 'at %s:%d, %s: %r' % (self.filename, self.lineno, + self.msg, self.line) + + +class ConfigParser(iniparser.BaseParser): + def __init__(self, filename, sections): + super(ConfigParser, self).__init__() + self.filename = filename + self.sections = sections + self.section = None + + def parse(self): + with open(self.filename) as f: + return super(ConfigParser, self).parse(f) + + def new_section(self, section): + self.section = section + self.sections.setdefault(self.section, {}) + + def assignment(self, key, value): + if not self.section: + raise self.error_no_section() + + self.sections[self.section].setdefault(key, []) + self.sections[self.section][key].append('\n'.join(value)) + + def parse_exc(self, msg, lineno, line=None): + return ParseError(msg, lineno, line, self.filename) + + def error_no_section(self): + return self.parse_exc('Section must be started before assignment', + self.lineno) + + +class MultiConfigParser(object): + def __init__(self): + self.sections = {} + + def read(self, config_files): + read_ok = [] + + for filename in config_files: + parser = ConfigParser(filename, self.sections) + + try: + parser.parse() + except IOError: + continue + + read_ok.append(filename) + + return read_ok + + def get(self, section, name): + return self.sections[section][name] + + class ConfigOpts(collections.Mapping): """ @@ -948,15 +1028,22 @@ class ConfigOpts(collections.Mapping): logger.log(lvl, "config files: %s", self.config_file) logger.log(lvl, "=" * 80) + def _sanitize(opt, value): + """Obfuscate values of options declared secret""" + return value if not opt.secret else '*' * len(str(value)) + for opt_name in sorted(self._opts): - logger.log(lvl, "%-30s = %s", opt_name, getattr(self, opt_name)) + opt = self._get_opt_info(opt_name)['opt'] + logger.log(lvl, "%-30s = %s", opt_name, + _sanitize(opt, getattr(self, opt_name))) for group_name in self._groups: group_attr = self.GroupAttr(self, self._get_group(group_name)) for opt_name in sorted(self._groups[group_name]._opts): + opt = self._get_opt_info(opt_name, group_name)['opt'] logger.log(lvl, "%-30s = %s", "%s.%s" % (group_name, opt_name), - getattr(group_attr, opt_name)) + _sanitize(opt, getattr(group_attr, opt_name))) logger.log(lvl, "*" * 80) @@ -986,20 +1073,31 @@ class ConfigOpts(collections.Mapping): if override is not None: return override + values = [] if self._cparser is not None: section = group.name if group is not None else 'DEFAULT' try: - return opt._get_from_config_parser(self._cparser, section) - except (ConfigParser.NoOptionError, - ConfigParser.NoSectionError): + value = opt._get_from_config_parser(self._cparser, section) + except KeyError: pass - except ValueError, ve: + except ValueError as ve: raise ConfigFileValueError(str(ve)) + else: + if not opt.multi: + # No need to continue since the last value wins + return value[-1] + values.extend(value) name = name if group is None else group.name + '_' + name - value = self._cli_values.get(name, None) + value = self._cli_values.get(name) if value is not None: - return value + if not opt.multi: + return value + + return value + values + + if values: + return values if default is not None: return default @@ -1069,12 +1167,12 @@ class ConfigOpts(collections.Mapping): :raises: ConfigFilesNotFoundError, ConfigFileParseError """ - self._cparser = ConfigParser.SafeConfigParser() + self._cparser = MultiConfigParser() try: read_ok = self._cparser.read(config_files) - except ConfigParser.ParsingError, cpe: - raise ConfigFileParseError(cpe.filename, cpe.message) + except iniparser.ParseError as pe: + raise ConfigFileParseError(pe.filename, str(pe)) if read_ok != config_files: not_read_ok = filter(lambda f: f not in read_ok, config_files) diff --git a/heat/openstack/common/iniparser.py b/heat/openstack/common/iniparser.py new file mode 100644 index 0000000000..53ca023343 --- /dev/null +++ b/heat/openstack/common/iniparser.py @@ -0,0 +1,126 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class ParseError(Exception): + def __init__(self, message, lineno, line): + self.msg = message + self.line = line + self.lineno = lineno + + def __str__(self): + return 'at line %d, %s: %r' % (self.lineno, self.msg, self.line) + + +class BaseParser(object): + lineno = 0 + parse_exc = ParseError + + def _assignment(self, key, value): + self.assignment(key, value) + return None, [] + + def _get_section(self, line): + if line[-1] != ']': + return self.error_no_section_end_bracket(line) + if len(line) <= 2: + return self.error_no_section_name(line) + + return line[1:-1] + + def _split_key_value(self, line): + colon = line.find(':') + equal = line.find('=') + if colon < 0 and equal < 0: + return self.error_invalid_assignment(line) + + if colon < 0 or (equal >= 0 and equal < colon): + key, value = line[:equal], line[equal + 1:] + else: + key, value = line[:colon], line[colon + 1:] + + return key.strip(), [value.strip()] + + def parse(self, lineiter): + key = None + value = [] + + for line in lineiter: + self.lineno += 1 + + line = line.rstrip() + if not line: + # Blank line, ends multi-line values + if key: + key, value = self._assignment(key, value) + continue + elif line[0] in (' ', '\t'): + # Continuation of previous assignment + if key is None: + self.error_unexpected_continuation(line) + else: + value.append(line.lstrip()) + continue + + if key: + # Flush previous assignment, if any + key, value = self._assignment(key, value) + + if line[0] == '[': + # Section start + section = self._get_section(line) + if section: + self.new_section(section) + elif line[0] in '#;': + self.comment(line[1:].lstrip()) + else: + key, value = self._split_key_value(line) + if not key: + return self.error_empty_key(line) + + if key: + # Flush previous assignment, if any + self._assignment(key, value) + + def assignment(self, key, value): + """Called when a full assignment is parsed""" + raise NotImplementedError() + + def new_section(self, section): + """Called when a new section is started""" + raise NotImplementedError() + + def comment(self, comment): + """Called when a comment is parsed""" + pass + + def error_invalid_assignment(self, line): + raise self.parse_exc("No ':' or '=' found in assignment", + self.lineno, line) + + def error_empty_key(self, line): + raise self.parse_exc('Key cannot be empty', self.lineno, line) + + def error_unexpected_continuation(self, line): + raise self.parse_exc('Unexpected continuation line', + self.lineno, line) + + def error_no_section_end_bracket(self, line): + raise self.parse_exc('Invalid section (must end with ])', + self.lineno, line) + + def error_no_section_name(self, line): + raise self.parse_exc('Empty section name', self.lineno, line) diff --git a/heat/local.py b/heat/openstack/common/local.py similarity index 100% rename from heat/local.py rename to heat/openstack/common/local.py diff --git a/heat/rpc/amqp.py b/heat/rpc/amqp.py index b4e548b3ca..c05f0be764 100644 --- a/heat/rpc/amqp.py +++ b/heat/rpc/amqp.py @@ -37,18 +37,18 @@ from eventlet import pools from heat import context from heat.common import exception from heat.common import config -from heat import local +from heat.openstack.common import local + import heat.rpc.common as rpc_common LOG = logging.getLogger(__name__) -FLAGS = config.FLAGS class Pool(pools.Pool): """Class that implements a Pool of Connections.""" def __init__(self, *args, **kwargs): self.connection_cls = kwargs.pop("connection_cls", None) - kwargs.setdefault("max_size", FLAGS.rpc_conn_pool_size) + kwargs.setdefault("max_size", config.FLAGS.rpc_conn_pool_size) kwargs.setdefault("order_as_stack", True) super(Pool, self).__init__(*args, **kwargs) @@ -206,7 +206,7 @@ class ProxyCallback(object): def __init__(self, proxy, connection_pool): self.proxy = proxy - self.pool = greenpool.GreenPool(FLAGS.rpc_thread_pool_size) + self.pool = greenpool.GreenPool(config.FLAGS.rpc_thread_pool_size) self.connection_pool = connection_pool def __call__(self, message_data): @@ -267,7 +267,7 @@ class MulticallWaiter(object): def __init__(self, connection, timeout): self._connection = connection self._iterator = connection.iterconsume( - timeout=timeout or FLAGS.rpc_response_timeout) + timeout=timeout or config.FLAGS.rpc_response_timeout) self._result = None self._done = False self._got_ending = False diff --git a/openstack-common.conf b/openstack-common.conf index 3a188982d4..6d10b5a971 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,7 +1,7 @@ [DEFAULT] # The list of modules to copy from openstack-common -modules=cfg +modules=cfg,local,iniparser # The base module to hold the copy of openstack.common base=heat