Update openstack-common

Change-Id: I5af06e0d44a69b9f968fce91db441157a69ea9c7
Signed-off-by: Angus Salkeld <asalkeld@redhat.com>
This commit is contained in:
Angus Salkeld 2012-06-06 15:15:15 +10:00
parent 7e81a8a8f9
commit a724259609
4 changed files with 353 additions and 106 deletions

View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 Red Hat, Inc.
# Copyright 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -90,16 +90,21 @@ the purposes of --help and CLI arg validation)::
def add_common_opts(conf):
conf.register_cli_opts(cli_opts)
The config manager has a single CLI option defined by default, --config-file::
The config manager has two CLI options defined by default, --config-file
and --config-dir::
class ConfigOpts(object):
config_file_opt = MultiStrOpt('config-file',
...
def __call__(self, ...):
def __init__(self, ...):
...
self.register_cli_opt(self.config_file_opt)
opts = [
MultiStrOpt('config-file',
...),
StrOpt('config-dir',
...),
]
self.register_cli_opts(opts)
Option values are parsed from any supplied config files using
openstack.common.iniparser. If none are specified, a default set is used
@ -144,6 +149,14 @@ Options can be registered as belonging to a group::
conf.register_opt(rabbit_host_opt, group=rabbit_group)
conf.register_opt(rabbit_port_opt, group='rabbit')
If it no group attributes are required other than the group name, the group
need not be explicitly registered e.g.
def register_rabbit_opts(conf):
# The group will automatically be created, equivalent calling::
# conf.register_group(OptGroup(name='rabbit'))
conf.register_opt(rabbit_port_opt, group='rabbit')
If no group is specified, options belong to the 'DEFAULT' section of config
files::
@ -208,6 +221,9 @@ as the leftover arguments, but will instead return::
i.e. argument parsing is stopped at the first non-option argument.
Options may be declared as required so that an error is raised if the user
does not supply a value for the option.
Options may be declared as secret so that their values are not leaked into
log files:
@ -217,11 +233,28 @@ log files:
...
]
This module also contains a global instance of the CommonConfigOpts class
in order to support a common usage pattern in OpenStack:
from openstack.common import cfg
opts = [
cfg.StrOpt('bind_host' default='0.0.0.0'),
cfg.IntOpt('bind_port', default=9292),
]
CONF = cfg.CONF
CONF.register_opts(opts)
def start(server, app):
server.start(app, CONF.bind_port, CONF.bind_host)
"""
import collections
import copy
import functools
import glob
import optparse
import os
import string
@ -285,6 +318,21 @@ class DuplicateOptError(Error):
return "duplicate option: %s" % self.opt_name
class RequiredOptError(Error):
"""Raised if an option is required but no value is supplied by the user."""
def __init__(self, opt_name, group=None):
self.opt_name = opt_name
self.group = group
def __str__(self):
if self.group is None:
return "value required for option: %s" % self.opt_name
else:
return "value required for option: %s.%s" % (self.group.name,
self.opt_name)
class TemplateSubstitutionError(Error):
"""Raised if an error occurs substituting a variable in an opt value."""
@ -319,6 +367,52 @@ class ConfigFileValueError(Error):
pass
def _get_config_dirs(project=None):
"""Return a list of directors where config files may be located.
:param project: an optional project name
If a project is specified, following directories are returned::
~/.${project}/
~/
/etc/${project}/
/etc/
Otherwise, these directories::
~/
/etc/
"""
fix_path = lambda p: os.path.abspath(os.path.expanduser(p))
cfg_dirs = [
fix_path(os.path.join('~', '.' + project)) if project else None,
fix_path('~'),
os.path.join('/etc', project) if project else None,
'/etc'
]
return filter(bool, cfg_dirs)
def _search_dirs(dirs, basename, extension=""):
"""Search a list of directories for a given filename.
Iterator over the supplied directories, returning the first file
found with the supplied name and extension.
:param dirs: a list of directories
:param basename: the filename, e.g. 'glance-api'
:param extension: the file extension, e.g. '.conf'
:returns: the path to a matching file, or None
"""
for d in dirs:
path = os.path.join(d, '%s%s' % (basename, extension))
if os.path.exists(path):
return path
def find_config_files(project=None, prog=None, extension='.conf'):
"""Return a list of default configuration files.
@ -347,26 +441,12 @@ def find_config_files(project=None, prog=None, extension='.conf'):
if prog is None:
prog = os.path.basename(sys.argv[0])
fix_path = lambda p: os.path.abspath(os.path.expanduser(p))
cfg_dirs = [
fix_path(os.path.join('~', '.' + project)) if project else None,
fix_path('~'),
os.path.join('/etc', project) if project else None,
'/etc'
]
cfg_dirs = filter(bool, cfg_dirs)
def search_dirs(dirs, basename, extension):
for d in dirs:
path = os.path.join(d, '%s%s' % (basename, extension))
if os.path.exists(path):
return path
cfg_dirs = _get_config_dirs(project)
config_files = []
if project:
config_files.append(search_dirs(cfg_dirs, project, extension))
config_files.append(search_dirs(cfg_dirs, prog, extension))
config_files.append(_search_dirs(cfg_dirs, project, extension))
config_files.append(_search_dirs(cfg_dirs, prog, extension))
return filter(bool, config_files)
@ -414,7 +494,7 @@ class Opt(object):
multi = False
def __init__(self, name, dest=None, short=None, default=None,
metavar=None, help=None, secret=False):
metavar=None, help=None, secret=False, required=False):
"""Construct an Opt object.
The only required parameter is the option's name. However, it is
@ -427,6 +507,7 @@ class Opt(object):
:param metavar: the option argument to show in --help
:param help: an explanation of how the option is used
:param secret: true iff the value should be obfuscated in log output
:param required: true iff a value must be supplied for this option
"""
self.name = name
if dest is None:
@ -438,6 +519,7 @@ class Opt(object):
self.metavar = metavar
self.help = help
self.secret = secret
self.required = required
def _get_from_config_parser(self, cparser, section):
"""Retrieves the option value from a MultiConfigParser object.
@ -702,6 +784,14 @@ class OptGroup(object):
return True
def _unregister_opt(self, opt):
"""Remove an opt from this group.
:param opt: an Opt object
"""
if opt.dest in self._opts:
del self._opts[opt.dest]
def _get_optparse_group(self, parser):
"""Build an optparse.OptionGroup for this group."""
if self._optparse_group is None:
@ -709,6 +799,10 @@ class OptGroup(object):
self.help)
return self._optparse_group
def _clear(self):
"""Clear this group's option parsing state."""
self._optparse_group = None
class ParseError(iniparser.ParseError):
def __init__(self, msg, lineno, line, filename):
@ -783,57 +877,59 @@ class ConfigOpts(collections.Mapping):
the values of options.
"""
def __init__(self,
project=None,
prog=None,
version=None,
usage=None,
default_config_files=None):
"""Construct a ConfigOpts object.
def __init__(self):
"""Construct a ConfigOpts object."""
self._opts = {} # dict of dicts of (opt:, override:, default:)
self._groups = {}
Automatically registers the --config-file option with either a supplied
list of default config files, or a list from find_config_files().
self._args = None
self._oparser = None
self._cparser = None
self._cli_values = {}
self.__cache = {}
self._config_opts = []
self._disable_interspersed_args = False
:param project: the toplevel project name, used to locate config files
:param prog: the name of the program (defaults to sys.argv[0] basename)
:param version: the program version (for --version)
:param usage: a usage string (%prog will be expanded)
:param default_config_files: config files to use by default
"""
def _setup(self, project, prog, version, usage, default_config_files):
"""Initialize a ConfigOpts object for option parsing."""
if prog is None:
prog = os.path.basename(sys.argv[0])
if default_config_files is None:
default_config_files = find_config_files(project, prog)
self._oparser = optparse.OptionParser(prog=prog,
version=version,
usage=usage)
if self._disable_interspersed_args:
self._oparser.disable_interspersed_args()
self._config_opts = [
MultiStrOpt('config-file',
default=default_config_files,
metavar='PATH',
help='Path to a config file to use. Multiple config '
'files can be specified, with values in later '
'files taking precedence. The default files '
' used are: %s' % (default_config_files, )),
StrOpt('config-dir',
metavar='DIR',
help='Path to a config directory to pull *.conf '
'files from. This file set is sorted, so as to '
'provide a predictable parse order if individual '
'options are over-ridden. The set is parsed after '
'the file(s), if any, specified via --config-file, '
'hence over-ridden options in the directory take '
'precedence.'),
]
self.register_cli_opts(self._config_opts)
self.project = project
self.prog = prog
self.version = version
self.usage = usage
self.default_config_files = default_config_files
self._opts = {} # dict of dicts of (opt:, override:, default:)
self._groups = {}
self._args = None
self._cli_values = {}
self._oparser = optparse.OptionParser(prog=self.prog,
version=self.version,
usage=self.usage)
self._cparser = None
self.__cache = {}
self.register_cli_opt(
MultiStrOpt('config-file',
default=self.default_config_files,
metavar='PATH',
help='Path to a config file to use. Multiple config '
'files can be specified, with values in later '
'files taking precedence. The default files used '
'are: %s' % (self.default_config_files, )))
def __clear_cache(f):
@functools.wraps(f)
def __inner(self, *args, **kwargs):
@ -843,7 +939,13 @@ class ConfigOpts(collections.Mapping):
return __inner
def __call__(self, args=None):
def __call__(self,
args=None,
project=None,
prog=None,
version=None,
usage=None,
default_config_files=None):
"""Parse command line arguments and config files.
Calling a ConfigOpts object causes the supplied command line arguments
@ -853,22 +955,34 @@ class ConfigOpts(collections.Mapping):
The object may be called multiple times, each time causing the previous
set of values to be overwritten.
:params args: command line arguments (defaults to sys.argv[1:])
Automatically registers the --config-file option with either a supplied
list of default config files, or a list from find_config_files().
If the --config-dir option is set, any *.conf files from this
directory are pulled in, after all the file(s) specified by the
--config-file option.
:param args: command line arguments (defaults to sys.argv[1:])
:param project: the toplevel project name, used to locate config files
:param prog: the name of the program (defaults to sys.argv[0] basename)
:param version: the program version (for --version)
:param usage: a usage string (%prog will be expanded)
:param default_config_files: config files to use by default
:returns: the list of arguments left over after parsing options
:raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError
:raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError,
RequiredOptError, DuplicateOptError
"""
self.reset()
self.clear()
self._args = args
self._setup(project, prog, version, usage, default_config_files)
(values, args) = self._oparser.parse_args(self._args)
self._cli_values, leftovers = self._parse_cli_opts(args)
self._cli_values = vars(values)
self._parse_config_files()
if self.config_file:
self._parse_config_files(self.config_file)
self._check_required_opts()
return args
return leftovers
def __getattr__(self, name):
"""Look up an option value and perform string substitution.
@ -896,12 +1010,21 @@ class ConfigOpts(collections.Mapping):
"""Return the number of options and option groups."""
return len(self._opts) + len(self._groups)
@__clear_cache
def reset(self):
"""Reset the state of the object to before it was called."""
"""Clear the object state and unset overrides and defaults."""
self._unset_defaults_and_overrides()
self.clear()
@__clear_cache
def clear(self):
"""Clear the state of the object to before it was called."""
self._args = None
self._cli_values = None
self._cli_values.clear()
self._oparser = None
self._cparser = None
self.unregister_opts(self._config_opts)
for group in self._groups.values():
group._clear()
@__clear_cache
def register_opt(self, opt, group=None):
@ -917,7 +1040,7 @@ class ConfigOpts(collections.Mapping):
:raises: DuplicateOptError
"""
if group is not None:
return self._get_group(group)._register_opt(opt)
return self._get_group(group, autocreate=True)._register_opt(opt)
if _is_opt_registered(self._opts, opt):
return False
@ -948,15 +1071,7 @@ class ConfigOpts(collections.Mapping):
if self._args is not None:
raise ArgsAlreadyParsedError("cannot register CLI option")
if not self.register_opt(opt, group, clear_cache=False):
return False
if group is not None:
group = self._get_group(group)
opt._add_to_cli(self._oparser, group)
return True
return self.register_opt(opt, group, clear_cache=False)
@__clear_cache
def register_cli_opts(self, opts, group=None):
@ -977,6 +1092,28 @@ class ConfigOpts(collections.Mapping):
self._groups[group.name] = copy.copy(group)
@__clear_cache
def unregister_opt(self, opt, group=None):
"""Unregister an option.
:param opt: an Opt object
:param group: an optional OptGroup object or group name
:raises: ArgsAlreadyParsedError, NoSuchGroupError
"""
if self._args is not None:
raise ArgsAlreadyParsedError("reset before unregistering options")
if group is not None:
self._get_group(group)._unregister_opt(opt)
elif opt.dest in self._opts:
del self._opts[opt.dest]
@__clear_cache
def unregister_opts(self, opts, group=None):
"""Unregister multiple CLI option schemas at once."""
for opt in opts:
self.unregister_opt(opt, group, clear_cache=False)
@__clear_cache
def set_override(self, name, override, group=None):
"""Override an opt value.
@ -1007,6 +1144,25 @@ class ConfigOpts(collections.Mapping):
opt_info = self._get_opt_info(name, group)
opt_info['default'] = default
def _all_opt_infos(self):
"""A generator function for iteration opt infos."""
for info in self._opts.values():
yield info, None
for group in self._groups.values():
for info in group._opts.values():
yield info, group
def _all_opts(self):
"""A generator function for iteration opts."""
for info, group in self._all_opt_infos():
yield info['opt'], group
def _unset_defaults_and_overrides(self):
"""Unset any default or override on all options."""
for info, group in self._all_opt_infos():
info['default'] = None
info['override'] = None
def disable_interspersed_args(self):
"""Set parsing to stop on the first non-option.
@ -1024,13 +1180,41 @@ class ConfigOpts(collections.Mapping):
i.e. argument parsing is stopped at the first non-option argument.
"""
self._oparser.disable_interspersed_args()
self._disable_interspersed_args = True
def enable_interspersed_args(self):
"""Set parsing to not stop on the first non-option.
This it the default behaviour."""
self._oparser.enable_interspersed_args()
self._disable_interspersed_args = False
def find_file(self, name):
"""Locate a file located alongside the config files.
Search for a file with the supplied basename in the directories
which we have already loaded config files from and other known
configuration directories.
The directory, if any, supplied by the config_dir option is
searched first. Then the config_file option is iterated over
and each of the base directories of the config_files values
are searched. Failing both of these, the standard directories
searched by the module level find_config_files() function is
used. The first matching file is returned.
:param basename: the filename, e.g. 'policy.json'
:returns: the path to a matching file, or None
"""
dirs = []
if self.config_dir:
dirs.append(self.config_dir)
for cf in reversed(self.config_file):
dirs.append(os.path.dirname(cf))
dirs.extend(_get_config_dirs(self.project))
return _search_dirs(dirs, name)
def log_opt_values(self, logger, lvl):
"""Log the value of all registered opts.
@ -1100,7 +1284,7 @@ class ConfigOpts(collections.Mapping):
return self.GroupAttr(self, self._get_group(name))
info = self._get_opt_info(name, group)
default, opt, override = map(lambda k: info[k], sorted(info.keys()))
default, opt, override = [info[k] for k in sorted(info.keys())]
if override is not None:
return override
@ -1153,7 +1337,7 @@ class ConfigOpts(collections.Mapping):
else:
return value
def _get_group(self, group_or_name):
def _get_group(self, group_or_name, autocreate=False):
"""Looks up a OptGroup object.
Helper function to return an OptGroup given a parameter which can
@ -1164,16 +1348,18 @@ class ConfigOpts(collections.Mapping):
the API have access to.
:param group_or_name: the group's name or the OptGroup object itself
:param autocreate: whether to auto-create the group if it's not found
:raises: NoSuchGroupError
"""
if isinstance(group_or_name, OptGroup):
group_name = group_or_name.name
else:
group_name = group_or_name
group = group_or_name if isinstance(group_or_name, OptGroup) else None
group_name = group.name if group else group_or_name
if not group_name in self._groups:
if not group is None or not autocreate:
raise NoSuchGroupError(group_name)
self.register_group(OptGroup(name=group_name))
return self._groups[group_name]
def _get_opt_info(self, opt_name, group=None):
@ -1194,11 +1380,17 @@ class ConfigOpts(collections.Mapping):
return opts[opt_name]
def _parse_config_files(self, config_files):
"""Parse the supplied configuration files.
def _parse_config_files(self):
"""Parse the config files from --config-file and --config-dir.
:raises: ConfigFilesNotFoundError, ConfigFileParseError
"""
config_files = list(self.config_file)
if self.config_dir:
config_dir_glob = os.path.join(self.config_dir, '*.conf')
config_files += sorted(glob.glob(config_dir_glob))
self._cparser = MultiConfigParser()
try:
@ -1210,6 +1402,42 @@ class ConfigOpts(collections.Mapping):
not_read_ok = filter(lambda f: f not in read_ok, config_files)
raise ConfigFilesNotFoundError(not_read_ok)
def _check_required_opts(self):
"""Check that all opts marked as required have values specified.
:raises: RequiredOptError
"""
for info, group in self._all_opt_infos():
default, opt, override = [info[k] for k in sorted(info.keys())]
if opt.required:
if (default is not None or
override is not None):
continue
if self._get(opt.name, group) is None:
raise RequiredOptError(opt.name, group)
def _parse_cli_opts(self, args):
"""Parse command line options.
Initializes the command line option parser and parses the supplied
command line arguments.
:param args: the command line arguments
:returns: a dict of parsed option values
:raises: SystemExit, DuplicateOptError
"""
self._args = args
for opt, group in self._all_opts():
opt._add_to_cli(self._oparser, group)
values, leftovers = self._oparser.parse_args(args)
return vars(values), leftovers
class GroupAttr(collections.Mapping):
"""
@ -1324,7 +1552,10 @@ class CommonConfigOpts(ConfigOpts):
help='syslog facility to receive log lines')
]
def __init__(self, **kwargs):
super(CommonConfigOpts, self).__init__(**kwargs)
def __init__(self):
super(CommonConfigOpts, self).__init__()
self.register_cli_opts(self.common_cli_opts)
self.register_cli_opts(self.logging_cli_opts)
CONF = CommonConfigOpts()

View File

@ -21,8 +21,6 @@ Import related utilities and helper functions.
import sys
from heat.openstack.common import exception
def import_class(import_str):
"""Returns a class from a string including module and class"""
@ -30,8 +28,9 @@ def import_class(import_str):
try:
__import__(mod_str)
return getattr(sys.modules[mod_str], class_str)
except (ImportError, ValueError, AttributeError):
raise exception.NotFound('Class %s cannot be found' % class_str)
except (ImportError, ValueError, AttributeError), exc:
raise ImportError('Class %s cannot be found (%s)' %
(class_str, str(exc)))
def import_object(import_str, *args, **kwargs):

View File

@ -61,9 +61,19 @@ def parse_requirements(requirements_files=['requirements.txt',
'tools/pip-requires']):
requirements = []
for line in get_reqs_from_files(requirements_files):
# For the requirements list, we need to inject only the portion
# after egg= so that distutils knows the package it's looking for
# such as:
# -e git://github.com/openstack/nova/master#egg=nova
if re.match(r'\s*-e\s+', line):
requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1',
line))
# such as:
# http://github.com/openstack/nova/zipball/master#egg=nova
elif re.match(r'\s*https?:', line):
requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1',
line))
# -f lines are for index locations, and don't get used here
elif re.match(r'\s*-f\s+', line):
pass
else:
@ -75,11 +85,18 @@ def parse_requirements(requirements_files=['requirements.txt',
def parse_dependency_links(requirements_files=['requirements.txt',
'tools/pip-requires']):
dependency_links = []
# dependency_links inject alternate locations to find packages listed
# in requirements
for line in get_reqs_from_files(requirements_files):
# skip comments and blank lines
if re.match(r'(\s*#)|(\s*$)', line):
continue
# lines with -e or -f need the whole line, minus the flag
if re.match(r'\s*-[ef]\s+', line):
dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line))
# lines that are only urls can go in unmolested
elif re.match(r'\s*https?:', line):
dependency_links.append(line)
return dependency_links
@ -137,8 +154,8 @@ def generate_authors():
new_authors = 'AUTHORS'
if os.path.isdir('.git'):
# don't include jenkins email address in AUTHORS file
git_log_cmd = "git log --format='%aN <%aE>' | sort -u | " \
"grep -v " + jenkins_email
git_log_cmd = ("git log --format='%aN <%aE>' | sort -u | "
"grep -v " + jenkins_email)
changelog = _run_shell_command(git_log_cmd)
mailmap = parse_mailmap()
with open(new_authors, 'w') as new_authors_fh:

View File

@ -30,7 +30,7 @@ TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
def isotime(at=None):
"""Stringify time in ISO 8601 format"""
if not at:
at = datetime.datetime.utcnow()
at = utcnow()
str = at.strftime(TIME_FORMAT)
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
str += ('Z' if tz == 'UTC' else tz)