Use from openstack-common

Signed-off-by: Angus Salkeld <>
Angus Salkeld 11 years ago
parent f4d4f60f8f
commit aff20aca99

@ -22,7 +22,7 @@
import copy
import logging
from heat import local
from heat.openstack.common import local
from heat.common import utils

@ -101,9 +101,9 @@ The config manager has a single CLI option defined by default, --config-file::
Option values are parsed from any supplied config files using SafeConfigParser.
If none are specified, a default set is used e.g. glance-api.conf and
Option values are parsed from any supplied config files using
openstack.common.iniparser. If none are specified, a default set is used
e.g. glance-api.conf and glance-common.conf::
@ -128,8 +128,8 @@ manager e.g.::
Options can be registered as belonging to a group::
rabbit_group = cfg.OptionGroup(name='rabbit',
title='RabbitMQ options')
rabbit_group = cfg.OptGroup(name='rabbit',
title='RabbitMQ options')
rabbit_host_opt = cfg.StrOpt('host',
@ -207,16 +207,27 @@ as the leftover arguments, but will instead return::
['cmd', '--debug', '/tmp/mything']
i.e. argument parsing is stopped at the first non-option argument.
Options may be declared as secret so that their values are not leaked into
log files:
opts = [
cfg.StrOpt('s3_store_access_key', secret=True),
cfg.StrOpt('s3_store_secret_key', secret=True),
import collections
import ConfigParser
import copy
import optparse
import os
import string
import sys
from heat.openstack.common import iniparser
class Error(Exception):
"""Base class for cfg exceptions."""
@ -398,9 +409,10 @@ class Opt(object):
an string explaining how the options value is used
multi = False
def __init__(self, name, dest=None, short=None,
default=None, metavar=None, help=None):
def __init__(self, name, dest=None, short=None, default=None,
metavar=None, help=None, secret=False):
"""Construct an Opt object.
The only required parameter is the option's name. However, it is
@ -412,6 +424,7 @@ class Opt(object):
:param default: the default value of the option
:param metavar: the option argument to show in --help
:param help: an explanation of how the option is used
:param secret: true iff the value should be obfuscated in log output
""" = name
if dest is None:
@ -422,9 +435,10 @@ class Opt(object):
self.default = default
self.metavar = metavar = help
self.secret = secret
def _get_from_config_parser(self, cparser, section):
"""Retrieves the option value from a ConfigParser object.
"""Retrieves the option value from a MultiConfigParser object.
This is the method ConfigOpts uses to look up the option value from
config files. Most opt types override this method in order to perform
@ -433,7 +447,7 @@ class Opt(object):
:param cparser: a ConfigParser object
:param section: a section name
return cparser.get(section, self.dest, raw=True)
return cparser.get(section, self.dest)
def _add_to_cli(self, parser, group=None):
"""Makes the option available in the command line interface.
@ -535,9 +549,19 @@ class BoolOpt(Opt):
1/0, yes/no, true/false or on/off.
_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
'0': False, 'no': False, 'false': False, 'off': False}
def _get_from_config_parser(self, cparser, section):
"""Retrieve the opt value as a boolean from ConfigParser."""
return cparser.getboolean(section, self.dest)
def convert_bool(v):
value = self._boolean_states.get(v.lower())
if value is None:
raise ValueError('Unexpected boolean value %r' % v)
return value
return [convert_bool(v) for v in cparser.get(section, self.dest)]
def _add_to_cli(self, parser, group=None):
"""Extends the base class method to add the --nooptname option."""
@ -564,7 +588,7 @@ class IntOpt(Opt):
def _get_from_config_parser(self, cparser, section):
"""Retrieve the opt value as a integer from ConfigParser."""
return cparser.getint(section, self.dest)
return [int(v) for v in cparser.get(section, self.dest)]
def _get_optparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for integer options."""
@ -578,7 +602,7 @@ class FloatOpt(Opt):
def _get_from_config_parser(self, cparser, section):
"""Retrieve the opt value as a float from ConfigParser."""
return cparser.getfloat(section, self.dest)
return [float(v) for v in cparser.get(section, self.dest)]
def _get_optparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for float options."""
@ -595,7 +619,7 @@ class ListOpt(Opt):
def _get_from_config_parser(self, cparser, section):
"""Retrieve the opt value as a list from ConfigParser."""
return cparser.get(section, self.dest).split(',')
return [v.split(',') for v in cparser.get(section, self.dest)]
def _get_optparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for list options."""
@ -617,14 +641,7 @@ class MultiStrOpt(Opt):
Multistr opt values are string opts which may be specified multiple times.
The opt value is a list containing all the string values specified.
def _get_from_config_parser(self, cparser, section):
"""Retrieve the opt value as a multistr from ConfigParser."""
# FIXME(markmc): values spread across the CLI and multiple
# config files should be appended
value = super(MultiStrOpt, self)._get_from_config_parser(cparser,
return value if value is None else [value]
multi = True
def _get_optparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for multi str options."""
@ -691,6 +708,69 @@ class OptGroup(object):
return self._optparse_group
class ParseError(iniparser.ParseError):
def __init__(self, msg, lineno, line, filename):
super(ParseError, self).__init__(msg, lineno, line)
self.filename = filename
def __str__(self):
return 'at %s:%d, %s: %r' % (self.filename, self.lineno,
self.msg, self.line)
class ConfigParser(iniparser.BaseParser):
def __init__(self, filename, sections):
super(ConfigParser, self).__init__()
self.filename = filename
self.sections = sections
self.section = None
def parse(self):
with open(self.filename) as f:
return super(ConfigParser, self).parse(f)
def new_section(self, section):
self.section = section
self.sections.setdefault(self.section, {})
def assignment(self, key, value):
if not self.section:
raise self.error_no_section()
self.sections[self.section].setdefault(key, [])
def parse_exc(self, msg, lineno, line=None):
return ParseError(msg, lineno, line, self.filename)
def error_no_section(self):
return self.parse_exc('Section must be started before assignment',
class MultiConfigParser(object):
def __init__(self):
self.sections = {}
def read(self, config_files):
read_ok = []
for filename in config_files:
parser = ConfigParser(filename, self.sections)
except IOError:
return read_ok
def get(self, section, name):
return self.sections[section][name]
class ConfigOpts(collections.Mapping):
@ -948,15 +1028,22 @@ class ConfigOpts(collections.Mapping):
logger.log(lvl, "config files: %s", self.config_file)
logger.log(lvl, "=" * 80)
def _sanitize(opt, value):
"""Obfuscate values of options declared secret"""
return value if not opt.secret else '*' * len(str(value))
for opt_name in sorted(self._opts):
logger.log(lvl, "%-30s = %s", opt_name, getattr(self, opt_name))
opt = self._get_opt_info(opt_name)['opt']
logger.log(lvl, "%-30s = %s", opt_name,
_sanitize(opt, getattr(self, opt_name)))
for group_name in self._groups:
group_attr = self.GroupAttr(self, self._get_group(group_name))
for opt_name in sorted(self._groups[group_name]._opts):
opt = self._get_opt_info(opt_name, group_name)['opt']
logger.log(lvl, "%-30s = %s",
"%s.%s" % (group_name, opt_name),
getattr(group_attr, opt_name))
_sanitize(opt, getattr(group_attr, opt_name)))
logger.log(lvl, "*" * 80)
@ -986,20 +1073,31 @@ class ConfigOpts(collections.Mapping):
if override is not None:
return override
values = []
if self._cparser is not None:
section = if group is not None else 'DEFAULT'
return opt._get_from_config_parser(self._cparser, section)
except (ConfigParser.NoOptionError,
value = opt._get_from_config_parser(self._cparser, section)
except KeyError:
except ValueError, ve:
except ValueError as ve:
raise ConfigFileValueError(str(ve))
if not opt.multi:
# No need to continue since the last value wins
return value[-1]
name = name if group is None else + '_' + name
value = self._cli_values.get(name, None)
value = self._cli_values.get(name)
if value is not None:
return value
if not opt.multi:
return value
return value + values
if values:
return values
if default is not None:
return default
@ -1069,12 +1167,12 @@ class ConfigOpts(collections.Mapping):
:raises: ConfigFilesNotFoundError, ConfigFileParseError
self._cparser = ConfigParser.SafeConfigParser()
self._cparser = MultiConfigParser()
read_ok =
except ConfigParser.ParsingError, cpe:
raise ConfigFileParseError(cpe.filename, cpe.message)
except iniparser.ParseError as pe:
raise ConfigFileParseError(pe.filename, str(pe))
if read_ok != config_files:
not_read_ok = filter(lambda f: f not in read_ok, config_files)

@ -0,0 +1,126 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 OpenStack LLC.
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class ParseError(Exception):
def __init__(self, message, lineno, line):
self.msg = message
self.line = line
self.lineno = lineno
def __str__(self):
return 'at line %d, %s: %r' % (self.lineno, self.msg, self.line)
class BaseParser(object):
lineno = 0
parse_exc = ParseError
def _assignment(self, key, value):
self.assignment(key, value)
return None, []
def _get_section(self, line):
if line[-1] != ']':
return self.error_no_section_end_bracket(line)
if len(line) <= 2:
return self.error_no_section_name(line)
return line[1:-1]
def _split_key_value(self, line):
colon = line.find(':')
equal = line.find('=')
if colon < 0 and equal < 0:
return self.error_invalid_assignment(line)
if colon < 0 or (equal >= 0 and equal < colon):
key, value = line[:equal], line[equal + 1:]
key, value = line[:colon], line[colon + 1:]
return key.strip(), [value.strip()]
def parse(self, lineiter):
key = None
value = []
for line in lineiter:
self.lineno += 1
line = line.rstrip()
if not line:
# Blank line, ends multi-line values
if key:
key, value = self._assignment(key, value)
elif line[0] in (' ', '\t'):
# Continuation of previous assignment
if key is None:
if key:
# Flush previous assignment, if any
key, value = self._assignment(key, value)
if line[0] == '[':
# Section start
section = self._get_section(line)
if section:
elif line[0] in '#;':
key, value = self._split_key_value(line)
if not key:
return self.error_empty_key(line)
if key:
# Flush previous assignment, if any
self._assignment(key, value)
def assignment(self, key, value):
"""Called when a full assignment is parsed"""
raise NotImplementedError()
def new_section(self, section):
"""Called when a new section is started"""
raise NotImplementedError()
def comment(self, comment):
"""Called when a comment is parsed"""
def error_invalid_assignment(self, line):
raise self.parse_exc("No ':' or '=' found in assignment",
self.lineno, line)
def error_empty_key(self, line):
raise self.parse_exc('Key cannot be empty', self.lineno, line)
def error_unexpected_continuation(self, line):
raise self.parse_exc('Unexpected continuation line',
self.lineno, line)
def error_no_section_end_bracket(self, line):
raise self.parse_exc('Invalid section (must end with ])',
self.lineno, line)
def error_no_section_name(self, line):
raise self.parse_exc('Empty section name', self.lineno, line)

@ -37,18 +37,18 @@ from eventlet import pools
from heat import context
from heat.common import exception
from heat.common import config
from heat import local
from heat.openstack.common import local
import heat.rpc.common as rpc_common
LOG = logging.getLogger(__name__)
FLAGS = config.FLAGS
class Pool(pools.Pool):
"""Class that implements a Pool of Connections."""
def __init__(self, *args, **kwargs):
self.connection_cls = kwargs.pop("connection_cls", None)
kwargs.setdefault("max_size", FLAGS.rpc_conn_pool_size)
kwargs.setdefault("max_size", config.FLAGS.rpc_conn_pool_size)
kwargs.setdefault("order_as_stack", True)
super(Pool, self).__init__(*args, **kwargs)
@ -206,7 +206,7 @@ class ProxyCallback(object):
def __init__(self, proxy, connection_pool):
self.proxy = proxy
self.pool = greenpool.GreenPool(FLAGS.rpc_thread_pool_size)
self.pool = greenpool.GreenPool(config.FLAGS.rpc_thread_pool_size)
self.connection_pool = connection_pool
def __call__(self, message_data):
@ -267,7 +267,7 @@ class MulticallWaiter(object):
def __init__(self, connection, timeout):
self._connection = connection
self._iterator = connection.iterconsume(
timeout=timeout or FLAGS.rpc_response_timeout)
timeout=timeout or config.FLAGS.rpc_response_timeout)
self._result = None
self._done = False
self._got_ending = False

@ -1,7 +1,7 @@
# The list of modules to copy from openstack-common
# The base module to hold the copy of openstack.common