Use local.py from openstack-common

Signed-off-by: Angus Salkeld <asalkeld@redhat.com>

heat/context.py

@@ -22,7 +22,7 @@
 import copy
 import logging
 
-from heat import local
+from heat.openstack.common import local
 from heat.common import utils
 
 

heat/openstack/common/cfg.py

@@ -101,9 +101,9 @@ The config manager has a single CLI option defined by default, --config-file::
             ...
             self.register_cli_opt(self.config_file_opt)
 
-Option values are parsed from any supplied config files using SafeConfigParser.
+Option values are parsed from any supplied config files using
-If none are specified, a default set is used e.g. glance-api.conf and
+openstack.common.iniparser. If none are specified, a default set is used
-glance-common.conf::
+e.g. glance-api.conf and glance-common.conf::
 
     glance-api.conf:
       [DEFAULT]
@@ -128,8 +128,8 @@ manager e.g.::
 
 Options can be registered as belonging to a group::
 
-    rabbit_group = cfg.OptionGroup(name='rabbit',
+    rabbit_group = cfg.OptGroup(name='rabbit',
-                                   title='RabbitMQ options')
+                                title='RabbitMQ options')
 
     rabbit_host_opt = cfg.StrOpt('host',
                                  default='localhost',
@@ -207,16 +207,27 @@ as the leftover arguments, but will instead return::
   ['cmd', '--debug', '/tmp/mything']
 
 i.e. argument parsing is stopped at the first non-option argument.
+
+Options may be declared as secret so that their values are not leaked into
+log files:
+
+     opts = [
+        cfg.StrOpt('s3_store_access_key', secret=True),
+        cfg.StrOpt('s3_store_secret_key', secret=True),
+        ...
+     ]
+
 """
 
 import collections
-import ConfigParser
 import copy
 import optparse
 import os
 import string
 import sys
 
+from heat.openstack.common import iniparser
+
 
 class Error(Exception):
     """Base class for cfg exceptions."""
@@ -398,9 +409,10 @@ class Opt(object):
       help:
         an string explaining how the options value is used
     """
+    multi = False
 
-    def __init__(self, name, dest=None, short=None,
+    def __init__(self, name, dest=None, short=None, default=None,
-                 default=None, metavar=None, help=None):
+                 metavar=None, help=None, secret=False):
         """Construct an Opt object.
 
         The only required parameter is the option's name. However, it is
@@ -412,6 +424,7 @@ class Opt(object):
         :param default: the default value of the option
         :param metavar: the option argument to show in --help
         :param help: an explanation of how the option is used
+        :param secret: true iff the value should be obfuscated in log output
         """
         self.name = name
         if dest is None:
@@ -422,9 +435,10 @@ class Opt(object):
         self.default = default
         self.metavar = metavar
         self.help = help
+        self.secret = secret
 
     def _get_from_config_parser(self, cparser, section):
-        """Retrieves the option value from a ConfigParser object.
+        """Retrieves the option value from a MultiConfigParser object.
 
         This is the method ConfigOpts uses to look up the option value from
         config files. Most opt types override this method in order to perform
@@ -433,7 +447,7 @@ class Opt(object):
         :param cparser: a ConfigParser object
         :param section: a section name
         """
-        return cparser.get(section, self.dest, raw=True)
+        return cparser.get(section, self.dest)
 
     def _add_to_cli(self, parser, group=None):
         """Makes the option available in the command line interface.
@@ -535,9 +549,19 @@ class BoolOpt(Opt):
     1/0, yes/no, true/false or on/off.
     """
 
+    _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
+                       '0': False, 'no': False, 'false': False, 'off': False}
+
     def _get_from_config_parser(self, cparser, section):
         """Retrieve the opt value as a boolean from ConfigParser."""
-        return cparser.getboolean(section, self.dest)
+        def convert_bool(v):
+            value = self._boolean_states.get(v.lower())
+            if value is None:
+                raise ValueError('Unexpected boolean value %r' % v)
+
+            return value
+
+        return [convert_bool(v) for v in cparser.get(section, self.dest)]
 
     def _add_to_cli(self, parser, group=None):
         """Extends the base class method to add the --nooptname option."""
@@ -564,7 +588,7 @@ class IntOpt(Opt):
 
     def _get_from_config_parser(self, cparser, section):
         """Retrieve the opt value as a integer from ConfigParser."""
-        return cparser.getint(section, self.dest)
+        return [int(v) for v in cparser.get(section, self.dest)]
 
     def _get_optparse_kwargs(self, group, **kwargs):
         """Extends the base optparse keyword dict for integer options."""
@@ -578,7 +602,7 @@ class FloatOpt(Opt):
 
     def _get_from_config_parser(self, cparser, section):
         """Retrieve the opt value as a float from ConfigParser."""
-        return cparser.getfloat(section, self.dest)
+        return [float(v) for v in cparser.get(section, self.dest)]
 
     def _get_optparse_kwargs(self, group, **kwargs):
         """Extends the base optparse keyword dict for float options."""
@@ -595,7 +619,7 @@ class ListOpt(Opt):
 
     def _get_from_config_parser(self, cparser, section):
         """Retrieve the opt value as a list from ConfigParser."""
-        return cparser.get(section, self.dest).split(',')
+        return [v.split(',') for v in cparser.get(section, self.dest)]
 
     def _get_optparse_kwargs(self, group, **kwargs):
         """Extends the base optparse keyword dict for list options."""
@@ -617,14 +641,7 @@ class MultiStrOpt(Opt):
     Multistr opt values are string opts which may be specified multiple times.
     The opt value is a list containing all the string values specified.
     """
-
+    multi = True
-    def _get_from_config_parser(self, cparser, section):
-        """Retrieve the opt value as a multistr from ConfigParser."""
-        # FIXME(markmc): values spread across the CLI and multiple
-        #                config files should be appended
-        value = super(MultiStrOpt, self)._get_from_config_parser(cparser,
-                                                                 section)
-        return value if value is None else [value]
 
     def _get_optparse_kwargs(self, group, **kwargs):
         """Extends the base optparse keyword dict for multi str options."""
@@ -691,6 +708,69 @@ class OptGroup(object):
         return self._optparse_group
 
 
+class ParseError(iniparser.ParseError):
+    def __init__(self, msg, lineno, line, filename):
+        super(ParseError, self).__init__(msg, lineno, line)
+        self.filename = filename
+
+    def __str__(self):
+        return 'at %s:%d, %s: %r' % (self.filename, self.lineno,
+                                     self.msg, self.line)
+
+
+class ConfigParser(iniparser.BaseParser):
+    def __init__(self, filename, sections):
+        super(ConfigParser, self).__init__()
+        self.filename = filename
+        self.sections = sections
+        self.section = None
+
+    def parse(self):
+        with open(self.filename) as f:
+            return super(ConfigParser, self).parse(f)
+
+    def new_section(self, section):
+        self.section = section
+        self.sections.setdefault(self.section, {})
+
+    def assignment(self, key, value):
+        if not self.section:
+            raise self.error_no_section()
+
+        self.sections[self.section].setdefault(key, [])
+        self.sections[self.section][key].append('\n'.join(value))
+
+    def parse_exc(self, msg, lineno, line=None):
+        return ParseError(msg, lineno, line, self.filename)
+
+    def error_no_section(self):
+        return self.parse_exc('Section must be started before assignment',
+                              self.lineno)
+
+
+class MultiConfigParser(object):
+    def __init__(self):
+        self.sections = {}
+
+    def read(self, config_files):
+        read_ok = []
+
+        for filename in config_files:
+            parser = ConfigParser(filename, self.sections)
+
+            try:
+                parser.parse()
+            except IOError:
+                continue
+
+            read_ok.append(filename)
+
+        return read_ok
+
+    def get(self, section, name):
+        return self.sections[section][name]
+
+
 class ConfigOpts(collections.Mapping):
 
     """
@@ -948,15 +1028,22 @@ class ConfigOpts(collections.Mapping):
         logger.log(lvl, "config files: %s", self.config_file)
         logger.log(lvl, "=" * 80)
 
+        def _sanitize(opt, value):
+            """Obfuscate values of options declared secret"""
+            return value if not opt.secret else '*' * len(str(value))
+
         for opt_name in sorted(self._opts):
-            logger.log(lvl, "%-30s = %s", opt_name, getattr(self, opt_name))
+            opt = self._get_opt_info(opt_name)['opt']
+            logger.log(lvl, "%-30s = %s", opt_name,
+                       _sanitize(opt, getattr(self, opt_name)))
 
         for group_name in self._groups:
             group_attr = self.GroupAttr(self, self._get_group(group_name))
             for opt_name in sorted(self._groups[group_name]._opts):
+                opt = self._get_opt_info(opt_name, group_name)['opt']
                 logger.log(lvl, "%-30s = %s",
                            "%s.%s" % (group_name, opt_name),
-                           getattr(group_attr, opt_name))
+                           _sanitize(opt, getattr(group_attr, opt_name)))
 
         logger.log(lvl, "*" * 80)
 
@@ -986,20 +1073,31 @@ class ConfigOpts(collections.Mapping):
         if override is not None:
             return override
 
+        values = []
         if self._cparser is not None:
             section = group.name if group is not None else 'DEFAULT'
             try:
-                return opt._get_from_config_parser(self._cparser, section)
+                value = opt._get_from_config_parser(self._cparser, section)
-            except (ConfigParser.NoOptionError,
+            except KeyError:
-                    ConfigParser.NoSectionError):
                 pass
-            except ValueError, ve:
+            except ValueError as ve:
                 raise ConfigFileValueError(str(ve))
+            else:
+                if not opt.multi:
+                    # No need to continue since the last value wins
+                    return value[-1]
+                values.extend(value)
 
         name = name if group is None else group.name + '_' + name
-        value = self._cli_values.get(name, None)
+        value = self._cli_values.get(name)
         if value is not None:
-            return value
+            if not opt.multi:
+                return value
+
+            return value + values
+
+        if values:
+            return values
 
         if default is not None:
             return default
@@ -1069,12 +1167,12 @@ class ConfigOpts(collections.Mapping):
 
         :raises: ConfigFilesNotFoundError, ConfigFileParseError
         """
-        self._cparser = ConfigParser.SafeConfigParser()
+        self._cparser = MultiConfigParser()
 
         try:
             read_ok = self._cparser.read(config_files)
-        except ConfigParser.ParsingError, cpe:
+        except iniparser.ParseError as pe:
-            raise ConfigFileParseError(cpe.filename, cpe.message)
+            raise ConfigFileParseError(pe.filename, str(pe))
 
         if read_ok != config_files:
             not_read_ok = filter(lambda f: f not in read_ok, config_files)

heat/openstack/common/iniparser.py

@@ -0,0 +1,126 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 OpenStack LLC.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+class ParseError(Exception):
+    def __init__(self, message, lineno, line):
+        self.msg = message
+        self.line = line
+        self.lineno = lineno
+
+    def __str__(self):
+        return 'at line %d, %s: %r' % (self.lineno, self.msg, self.line)
+
+
+class BaseParser(object):
+    lineno = 0
+    parse_exc = ParseError
+
+    def _assignment(self, key, value):
+        self.assignment(key, value)
+        return None, []
+
+    def _get_section(self, line):
+        if line[-1] != ']':
+            return self.error_no_section_end_bracket(line)
+        if len(line) <= 2:
+            return self.error_no_section_name(line)
+
+        return line[1:-1]
+
+    def _split_key_value(self, line):
+        colon = line.find(':')
+        equal = line.find('=')
+        if colon < 0 and equal < 0:
+            return self.error_invalid_assignment(line)
+
+        if colon < 0 or (equal >= 0 and equal < colon):
+            key, value = line[:equal], line[equal + 1:]
+        else:
+            key, value = line[:colon], line[colon + 1:]
+
+        return key.strip(), [value.strip()]
+
+    def parse(self, lineiter):
+        key = None
+        value = []
+
+        for line in lineiter:
+            self.lineno += 1
+
+            line = line.rstrip()
+            if not line:
+                # Blank line, ends multi-line values
+                if key:
+                    key, value = self._assignment(key, value)
+                continue
+            elif line[0] in (' ', '\t'):
+                # Continuation of previous assignment
+                if key is None:
+                    self.error_unexpected_continuation(line)
+                else:
+                    value.append(line.lstrip())
+                continue
+
+            if key:
+                # Flush previous assignment, if any
+                key, value = self._assignment(key, value)
+
+            if line[0] == '[':
+                # Section start
+                section = self._get_section(line)
+                if section:
+                    self.new_section(section)
+            elif line[0] in '#;':
+                self.comment(line[1:].lstrip())
+            else:
+                key, value = self._split_key_value(line)
+                if not key:
+                    return self.error_empty_key(line)
+
+        if key:
+            # Flush previous assignment, if any
+            self._assignment(key, value)
+
+    def assignment(self, key, value):
+        """Called when a full assignment is parsed"""
+        raise NotImplementedError()
+
+    def new_section(self, section):
+        """Called when a new section is started"""
+        raise NotImplementedError()
+
+    def comment(self, comment):
+        """Called when a comment is parsed"""
+        pass
+
+    def error_invalid_assignment(self, line):
+        raise self.parse_exc("No ':' or '=' found in assignment",
+                             self.lineno, line)
+
+    def error_empty_key(self, line):
+        raise self.parse_exc('Key cannot be empty', self.lineno, line)
+
+    def error_unexpected_continuation(self, line):
+        raise self.parse_exc('Unexpected continuation line',
+                             self.lineno, line)
+
+    def error_no_section_end_bracket(self, line):
+        raise self.parse_exc('Invalid section (must end with ])',
+                             self.lineno, line)
+
+    def error_no_section_name(self, line):
+        raise self.parse_exc('Empty section name', self.lineno, line)

heat/rpc/amqp.py

@@ -37,18 +37,18 @@ from eventlet import pools
 from heat import context
 from heat.common import exception
 from heat.common import config
-from heat import local
+from heat.openstack.common import local
+
 import heat.rpc.common as rpc_common
 
 LOG = logging.getLogger(__name__)
-FLAGS = config.FLAGS
 
 
 class Pool(pools.Pool):
     """Class that implements a Pool of Connections."""
     def __init__(self, *args, **kwargs):
         self.connection_cls = kwargs.pop("connection_cls", None)
-        kwargs.setdefault("max_size", FLAGS.rpc_conn_pool_size)
+        kwargs.setdefault("max_size", config.FLAGS.rpc_conn_pool_size)
         kwargs.setdefault("order_as_stack", True)
         super(Pool, self).__init__(*args, **kwargs)
 
@@ -206,7 +206,7 @@ class ProxyCallback(object):
 
     def __init__(self, proxy, connection_pool):
         self.proxy = proxy
-        self.pool = greenpool.GreenPool(FLAGS.rpc_thread_pool_size)
+        self.pool = greenpool.GreenPool(config.FLAGS.rpc_thread_pool_size)
         self.connection_pool = connection_pool
 
     def __call__(self, message_data):
@@ -267,7 +267,7 @@ class MulticallWaiter(object):
     def __init__(self, connection, timeout):
         self._connection = connection
         self._iterator = connection.iterconsume(
-                                timeout=timeout or FLAGS.rpc_response_timeout)
+                                timeout=timeout or config.FLAGS.rpc_response_timeout)
         self._result = None
         self._done = False
         self._got_ending = False

openstack-common.conf

@@ -1,7 +1,7 @@
 [DEFAULT]
 
 # The list of modules to copy from openstack-common
-modules=cfg
+modules=cfg,local,iniparser
 
 # The base module to hold the copy of openstack.common
 base=heat