Sync latest cfg from openstack-common

Syncs the following changes from stable/folsom:

 4dc2043 cfg: clean up None value handling
 513bd3a Allow set_default and set_override to use None
 0a36c92 Tilde expansion for --config-file and --config-dir
 038d597 Add import_opt() method to ConfigOpts
 90da88c Modifies _is_opt_registered fcn to check for duplicate opts
 3af0786 cfg: Fix typo in documentation
 9f93872 Update common code to support pep 1.3. bug 1014216
 fb0f00e Use 'is not None' instead of '!= None'
 d1ec5ce Add support to include config aliases

Also, exclude keystone/openstack/common from pep8 checks because
it is pep8-checked in openstack-common anyway and all pep8 fixes
would need to go into openstack-common first.

Change-Id: Ie6354251942648c16c2be270af2b95f3f45e27ed

keystone/openstack/common/cfg.py

@@ -42,8 +42,8 @@ Options can be strings, integers, floats, booleans, lists or 'multi strings'::
     osapi_compute_extension_opt = cfg.MultiStrOpt('osapi_compute_extension',
                                                   default=DEFAULT_EXTENSIONS)
 
-Option schemas are registered with with the config manager at runtime, but
+Option schemas are registered with the config manager at runtime, but before
-before the option is referenced::
+the option is referenced::
 
     class ExtensionManager(object):
 
@@ -367,6 +367,11 @@ class ConfigFileValueError(Error):
     pass
 
 
+def _fixpath(p):
+    """Apply tilde expansion and absolutization to a path."""
+    return os.path.abspath(os.path.expanduser(p))
+
+
 def _get_config_dirs(project=None):
     """Return a list of directors where config files may be located.
 
@@ -384,11 +389,9 @@ def _get_config_dirs(project=None):
       ~/
       /etc/
     """
-    fix_path = lambda p: os.path.abspath(os.path.expanduser(p))
-
     cfg_dirs = [
-        fix_path(os.path.join('~', '.' + project)) if project else None,
+        _fixpath(os.path.join('~', '.' + project)) if project else None,
-        fix_path('~'),
+        _fixpath('~'),
         os.path.join('/etc', project) if project else None,
         '/etc'
     ]
@@ -464,7 +467,7 @@ def _is_opt_registered(opts, opt):
     :raises: DuplicateOptError if a naming conflict is detected
     """
     if opt.dest in opts:
-        if opts[opt.dest]['opt'] is not opt:
+        if opts[opt.dest]['opt'] != opt:
             raise DuplicateOptError(opt.name)
         return True
     else:
@@ -494,7 +497,8 @@ class Opt(object):
     multi = False
 
     def __init__(self, name, dest=None, short=None, default=None,
-                 metavar=None, help=None, secret=False, required=False):
+                 metavar=None, help=None, secret=False, required=False,
+                 deprecated_name=None):
         """Construct an Opt object.
 
         The only required parameter is the option's name. However, it is
@@ -508,6 +512,7 @@ class Opt(object):
         :param help: an explanation of how the option is used
         :param secret: true iff the value should be obfuscated in log output
         :param required: true iff a value must be supplied for this option
+        :param deprecated_name: deprecated name option.  Acts like an alias
         """
         self.name = name
         if dest is None:
@@ -520,6 +525,13 @@ class Opt(object):
         self.help = help
         self.secret = secret
         self.required = required
+        if deprecated_name is not None:
+            self.deprecated_name = deprecated_name.replace('-', '_')
+        else:
+            self.deprecated_name = None
+
+    def __ne__(self, another):
+        return vars(self) != vars(another)
 
     def _get_from_config_parser(self, cparser, section):
         """Retrieves the option value from a MultiConfigParser object.
@@ -531,7 +543,13 @@ class Opt(object):
         :param cparser: a ConfigParser object
         :param section: a section name
         """
-        return cparser.get(section, self.dest)
+        return self._cparser_get_with_deprecated(cparser, section)
+
+    def _cparser_get_with_deprecated(self, cparser, section):
+        """If cannot find option as dest try deprecated_name alias."""
+        if self.deprecated_name is not None:
+            return cparser.get(section, [self.dest, self.deprecated_name])
+        return cparser.get(section, [self.dest])
 
     def _add_to_cli(self, parser, group=None):
         """Makes the option available in the command line interface.
@@ -546,9 +564,11 @@ class Opt(object):
         container = self._get_optparse_container(parser, group)
         kwargs = self._get_optparse_kwargs(group)
         prefix = self._get_optparse_prefix('', group)
-        self._add_to_optparse(container, self.name, self.short, kwargs, prefix)
+        self._add_to_optparse(container, self.name, self.short, kwargs, prefix,
+                              self.deprecated_name)
 
-    def _add_to_optparse(self, container, name, short, kwargs, prefix=''):
+    def _add_to_optparse(self, container, name, short, kwargs, prefix='',
+                         deprecated_name=None):
         """Add an option to an optparse parser or group.
 
         :param container: an optparse.OptionContainer object
@@ -561,6 +581,8 @@ class Opt(object):
         args = ['--' + prefix + name]
         if short:
             args += ['-' + short]
+        if deprecated_name:
+            args += ['--' + prefix + deprecated_name]
         for a in args:
             if container.has_option(a):
                 raise DuplicateOptError(a)
@@ -591,11 +613,9 @@ class Opt(object):
         dest = self.dest
         if group is not None:
             dest = group.name + '_' + dest
-        kwargs.update({
+        kwargs.update({'dest': dest,
-            'dest': dest,
                        'metavar': self.metavar,
-            'help': self.help,
+                       'help': self.help, })
-        })
         return kwargs
 
     def _get_optparse_prefix(self, prefix, group):
@@ -645,7 +665,8 @@ class BoolOpt(Opt):
 
             return value
 
-        return [convert_bool(v) for v in cparser.get(section, self.dest)]
+        return [convert_bool(v) for v in
+                self._cparser_get_with_deprecated(cparser, section)]
 
     def _add_to_cli(self, parser, group=None):
         """Extends the base class method to add the --nooptname option."""
@@ -658,7 +679,8 @@ class BoolOpt(Opt):
         kwargs = self._get_optparse_kwargs(group, action='store_false')
         prefix = self._get_optparse_prefix('no', group)
         kwargs["help"] = "The inverse of --" + self.name
-        self._add_to_optparse(container, self.name, None, kwargs, prefix)
+        self._add_to_optparse(container, self.name, None, kwargs, prefix,
+                              self.deprecated_name)
 
     def _get_optparse_kwargs(self, group, action='store_true', **kwargs):
         """Extends the base optparse keyword dict for boolean options."""
@@ -672,7 +694,8 @@ class IntOpt(Opt):
 
     def _get_from_config_parser(self, cparser, section):
         """Retrieve the opt value as a integer from ConfigParser."""
-        return [int(v) for v in cparser.get(section, self.dest)]
+        return [int(v) for v in self._cparser_get_with_deprecated(cparser,
+                section)]
 
     def _get_optparse_kwargs(self, group, **kwargs):
         """Extends the base optparse keyword dict for integer options."""
@@ -686,7 +709,8 @@ class FloatOpt(Opt):
 
     def _get_from_config_parser(self, cparser, section):
         """Retrieve the opt value as a float from ConfigParser."""
-        return [float(v) for v in cparser.get(section, self.dest)]
+        return [float(v) for v in
+                self._cparser_get_with_deprecated(cparser, section)]
 
     def _get_optparse_kwargs(self, group, **kwargs):
         """Extends the base optparse keyword dict for float options."""
@@ -703,7 +727,8 @@ class ListOpt(Opt):
 
     def _get_from_config_parser(self, cparser, section):
         """Retrieve the opt value as a list from ConfigParser."""
-        return [v.split(',') for v in cparser.get(section, self.dest)]
+        return [v.split(',') for v in
+                self._cparser_get_with_deprecated(cparser, section)]
 
     def _get_optparse_kwargs(self, group, **kwargs):
         """Extends the base optparse keyword dict for list options."""
@@ -732,6 +757,13 @@ class MultiStrOpt(Opt):
         return super(MultiStrOpt,
                      self)._get_optparse_kwargs(group, action='append')
 
+    def _cparser_get_with_deprecated(self, cparser, section):
+        """If cannot find option as dest try deprecated_name alias."""
+        if self.deprecated_name is not None:
+            return cparser.get(section, [self.dest, self.deprecated_name],
+                               multi=True)
+        return cparser.get(section, [self.dest], multi=True)
+
 
 class OptGroup(object):
 
@@ -780,7 +812,7 @@ class OptGroup(object):
         if _is_opt_registered(self._opts, opt):
             return False
 
-        self._opts[opt.dest] = {'opt': opt, 'override': None, 'default': None}
+        self._opts[opt.dest] = {'opt': opt}
 
         return True
 
@@ -846,25 +878,38 @@ class ConfigParser(iniparser.BaseParser):
 
 class MultiConfigParser(object):
     def __init__(self):
-        self.sections = {}
+        self.parsed = []
 
     def read(self, config_files):
         read_ok = []
 
         for filename in config_files:
-            parser = ConfigParser(filename, self.sections)
+            sections = {}
+            parser = ConfigParser(filename, sections)
 
             try:
                 parser.parse()
             except IOError:
                 continue
-
+            self.parsed.insert(0, sections)
             read_ok.append(filename)
 
         return read_ok
 
-    def get(self, section, name):
+    def get(self, section, names, multi=False):
-        return self.sections[section][name]
+        rvalue = []
+        for sections in self.parsed:
+            if section not in sections:
+                continue
+            for name in names:
+                if name in sections[section]:
+                    if multi:
+                        rvalue = sections[section][name] + rvalue
+                    else:
+                        return sections[section][name]
+        if multi and rvalue != []:
+            return rvalue
+        raise KeyError
 
 
 class ConfigOpts(collections.Mapping):
@@ -1045,7 +1090,7 @@ class ConfigOpts(collections.Mapping):
         if _is_opt_registered(self._opts, opt):
             return False
 
-        self._opts[opt.dest] = {'opt': opt, 'override': None, 'default': None}
+        self._opts[opt.dest] = {'opt': opt}
 
         return True
 
@@ -1114,6 +1159,25 @@ class ConfigOpts(collections.Mapping):
         for opt in opts:
             self.unregister_opt(opt, group, clear_cache=False)
 
+    def import_opt(self, name, module_str, group=None):
+        """Import an option definition from a module.
+
+        Import a module and check that a given option is registered.
+
+        This is intended for use with global configuration objects
+        like cfg.CONF where modules commonly register options with
+        CONF at module load time. If one module requires an option
+        defined by another module it can use this method to explicitly
+        declare the dependency.
+
+        :param name: the name/dest of the opt
+        :param module_str: the name of a module to import
+        :param group: an option OptGroup object or group name
+        :raises: NoSuchOptError, NoSuchGroupError
+        """
+        __import__(module_str)
+        self._get_opt_info(name, group)
+
     @__clear_cache
     def set_override(self, name, override, group=None):
         """Override an opt value.
@@ -1144,6 +1208,33 @@ class ConfigOpts(collections.Mapping):
         opt_info = self._get_opt_info(name, group)
         opt_info['default'] = default
 
+    @__clear_cache
+    def clear_override(self, name, group=None):
+        """Clear an override an opt value.
+
+        Clear a previously set override of the command line, config file
+        and default values of a given option.
+
+        :param name: the name/dest of the opt
+        :param group: an option OptGroup object or group name
+        :raises: NoSuchOptError, NoSuchGroupError
+        """
+        opt_info = self._get_opt_info(name, group)
+        opt_info.pop('override', None)
+
+    @__clear_cache
+    def clear_default(self, name, group=None):
+        """Clear an override an opt's default value.
+
+        Clear a previously set override of the default value of given option.
+
+        :param name: the name/dest of the opt
+        :param group: an option OptGroup object or group name
+        :raises: NoSuchOptError, NoSuchGroupError
+        """
+        opt_info = self._get_opt_info(name, group)
+        opt_info.pop('default', None)
+
     def _all_opt_infos(self):
         """A generator function for iteration opt infos."""
         for info in self._opts.values():
@@ -1160,8 +1251,8 @@ class ConfigOpts(collections.Mapping):
     def _unset_defaults_and_overrides(self):
         """Unset any default or override on all options."""
         for info, group in self._all_opt_infos():
-            info['default'] = None
+            info.pop('default', None)
-            info['override'] = None
+            info.pop('override', None)
 
     def disable_interspersed_args(self):
         """Set parsing to stop on the first non-option.
@@ -1207,10 +1298,10 @@ class ConfigOpts(collections.Mapping):
         """
         dirs = []
         if self.config_dir:
-            dirs.append(self.config_dir)
+            dirs.append(_fixpath(self.config_dir))
 
         for cf in reversed(self.config_file):
-            dirs.append(os.path.dirname(cf))
+            dirs.append(os.path.dirname(_fixpath(cf)))
 
         dirs.extend(_get_config_dirs(self.project))
 
@@ -1284,10 +1375,10 @@ class ConfigOpts(collections.Mapping):
             return self.GroupAttr(self, self._get_group(name))
 
         info = self._get_opt_info(name, group)
-        default, opt, override = [info[k] for k in sorted(info.keys())]
+        opt = info['opt']
 
-        if override is not None:
+        if 'override' in info:
-            return override
+            return info['override']
 
         values = []
         if self._cparser is not None:
@@ -1315,15 +1406,15 @@ class ConfigOpts(collections.Mapping):
         if values:
             return values
 
-        if default is not None:
+        if 'default' in info:
-            return default
+            return info['default']
 
         return opt.default
 
     def _substitute(self, value):
         """Perform string template substitution.
 
-        Substititue any template variables (e.g. $foo, ${bar}) in the supplied
+        Substitute any template variables (e.g. $foo, ${bar}) in the supplied
         string value(s) with opt values.
 
         :param value: the string value, or list of string values
@@ -1391,6 +1482,8 @@ class ConfigOpts(collections.Mapping):
             config_dir_glob = os.path.join(self.config_dir, '*.conf')
             config_files += sorted(glob.glob(config_dir_glob))
 
+        config_files = [_fixpath(p) for p in config_files]
+
         self._cparser = MultiConfigParser()
 
         try:
@@ -1408,10 +1501,10 @@ class ConfigOpts(collections.Mapping):
         :raises: RequiredOptError
         """
         for info, group in self._all_opt_infos():
-            default, opt, override = [info[k] for k in sorted(info.keys())]
+            opt = info['opt']
 
             if opt.required:
-                if (default is not None or override is not None):
+                if ('default' in info or 'override' in info):
                     continue
 
                 if self._get(opt.name, group) is None:

keystone/openstack/common/iniparser.py

@@ -52,7 +52,11 @@ class BaseParser(object):
         else:
             key, value = line[:colon], line[colon + 1:]
 
-        return key.strip(), [value.strip()]
+        value = value.strip()
+        if ((value and value[0] == value[-1]) and
+            (value[0] == "\"" or value[0] == "'")):
+            value = value[1:-1]
+        return key.strip(), [value]
 
     def parse(self, lineiter):
         key = None

tox.ini

@@ -15,7 +15,7 @@ commands = bash run_tests.sh -N --no-pep8
 
 [testenv:pep8]
 deps = pep8==1.3.3
-commands = pep8 --exclude=vcsversion.py,*.pyc --repeat --show-source bin doc httpd keystone setup.py tests
+commands = pep8 --exclude=vcsversion.py,*.pyc,openstack --repeat --show-source bin doc httpd keystone setup.py tests
 
 [tox:jenkins]
 downloadcache = ~/cache/pip