diff --git a/heat/openstack/common/cfg.py b/heat/openstack/common/cfg.py index e1e31147a7..b68c19b076 100644 --- a/heat/openstack/common/cfg.py +++ b/heat/openstack/common/cfg.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 Red Hat, Inc. +# Copyright 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -90,16 +90,21 @@ the purposes of --help and CLI arg validation):: def add_common_opts(conf): conf.register_cli_opts(cli_opts) -The config manager has a single CLI option defined by default, --config-file:: +The config manager has two CLI options defined by default, --config-file +and --config-dir:: class ConfigOpts(object): - config_file_opt = MultiStrOpt('config-file', - ... + def __call__(self, ...): - def __init__(self, ...): - ... - self.register_cli_opt(self.config_file_opt) + opts = [ + MultiStrOpt('config-file', + ...), + StrOpt('config-dir', + ...), + ] + + self.register_cli_opts(opts) Option values are parsed from any supplied config files using openstack.common.iniparser. If none are specified, a default set is used @@ -144,6 +149,14 @@ Options can be registered as belonging to a group:: conf.register_opt(rabbit_host_opt, group=rabbit_group) conf.register_opt(rabbit_port_opt, group='rabbit') +If it no group attributes are required other than the group name, the group +need not be explicitly registered e.g. + + def register_rabbit_opts(conf): + # The group will automatically be created, equivalent calling:: + # conf.register_group(OptGroup(name='rabbit')) + conf.register_opt(rabbit_port_opt, group='rabbit') + If no group is specified, options belong to the 'DEFAULT' section of config files:: @@ -208,6 +221,9 @@ as the leftover arguments, but will instead return:: i.e. argument parsing is stopped at the first non-option argument. +Options may be declared as required so that an error is raised if the user +does not supply a value for the option. + Options may be declared as secret so that their values are not leaked into log files: @@ -217,11 +233,28 @@ log files: ... ] +This module also contains a global instance of the CommonConfigOpts class +in order to support a common usage pattern in OpenStack: + + from openstack.common import cfg + + opts = [ + cfg.StrOpt('bind_host' default='0.0.0.0'), + cfg.IntOpt('bind_port', default=9292), + ] + + CONF = cfg.CONF + CONF.register_opts(opts) + + def start(server, app): + server.start(app, CONF.bind_port, CONF.bind_host) + """ import collections import copy import functools +import glob import optparse import os import string @@ -285,6 +318,21 @@ class DuplicateOptError(Error): return "duplicate option: %s" % self.opt_name +class RequiredOptError(Error): + """Raised if an option is required but no value is supplied by the user.""" + + def __init__(self, opt_name, group=None): + self.opt_name = opt_name + self.group = group + + def __str__(self): + if self.group is None: + return "value required for option: %s" % self.opt_name + else: + return "value required for option: %s.%s" % (self.group.name, + self.opt_name) + + class TemplateSubstitutionError(Error): """Raised if an error occurs substituting a variable in an opt value.""" @@ -319,6 +367,52 @@ class ConfigFileValueError(Error): pass +def _get_config_dirs(project=None): + """Return a list of directors where config files may be located. + + :param project: an optional project name + + If a project is specified, following directories are returned:: + + ~/.${project}/ + ~/ + /etc/${project}/ + /etc/ + + Otherwise, these directories:: + + ~/ + /etc/ + """ + fix_path = lambda p: os.path.abspath(os.path.expanduser(p)) + + cfg_dirs = [ + fix_path(os.path.join('~', '.' + project)) if project else None, + fix_path('~'), + os.path.join('/etc', project) if project else None, + '/etc' + ] + + return filter(bool, cfg_dirs) + + +def _search_dirs(dirs, basename, extension=""): + """Search a list of directories for a given filename. + + Iterator over the supplied directories, returning the first file + found with the supplied name and extension. + + :param dirs: a list of directories + :param basename: the filename, e.g. 'glance-api' + :param extension: the file extension, e.g. '.conf' + :returns: the path to a matching file, or None + """ + for d in dirs: + path = os.path.join(d, '%s%s' % (basename, extension)) + if os.path.exists(path): + return path + + def find_config_files(project=None, prog=None, extension='.conf'): """Return a list of default configuration files. @@ -347,26 +441,12 @@ def find_config_files(project=None, prog=None, extension='.conf'): if prog is None: prog = os.path.basename(sys.argv[0]) - fix_path = lambda p: os.path.abspath(os.path.expanduser(p)) - - cfg_dirs = [ - fix_path(os.path.join('~', '.' + project)) if project else None, - fix_path('~'), - os.path.join('/etc', project) if project else None, - '/etc' - ] - cfg_dirs = filter(bool, cfg_dirs) - - def search_dirs(dirs, basename, extension): - for d in dirs: - path = os.path.join(d, '%s%s' % (basename, extension)) - if os.path.exists(path): - return path + cfg_dirs = _get_config_dirs(project) config_files = [] if project: - config_files.append(search_dirs(cfg_dirs, project, extension)) - config_files.append(search_dirs(cfg_dirs, prog, extension)) + config_files.append(_search_dirs(cfg_dirs, project, extension)) + config_files.append(_search_dirs(cfg_dirs, prog, extension)) return filter(bool, config_files) @@ -414,7 +494,7 @@ class Opt(object): multi = False def __init__(self, name, dest=None, short=None, default=None, - metavar=None, help=None, secret=False): + metavar=None, help=None, secret=False, required=False): """Construct an Opt object. The only required parameter is the option's name. However, it is @@ -427,6 +507,7 @@ class Opt(object): :param metavar: the option argument to show in --help :param help: an explanation of how the option is used :param secret: true iff the value should be obfuscated in log output + :param required: true iff a value must be supplied for this option """ self.name = name if dest is None: @@ -438,6 +519,7 @@ class Opt(object): self.metavar = metavar self.help = help self.secret = secret + self.required = required def _get_from_config_parser(self, cparser, section): """Retrieves the option value from a MultiConfigParser object. @@ -702,6 +784,14 @@ class OptGroup(object): return True + def _unregister_opt(self, opt): + """Remove an opt from this group. + + :param opt: an Opt object + """ + if opt.dest in self._opts: + del self._opts[opt.dest] + def _get_optparse_group(self, parser): """Build an optparse.OptionGroup for this group.""" if self._optparse_group is None: @@ -709,6 +799,10 @@ class OptGroup(object): self.help) return self._optparse_group + def _clear(self): + """Clear this group's option parsing state.""" + self._optparse_group = None + class ParseError(iniparser.ParseError): def __init__(self, msg, lineno, line, filename): @@ -783,57 +877,59 @@ class ConfigOpts(collections.Mapping): the values of options. """ - def __init__(self, - project=None, - prog=None, - version=None, - usage=None, - default_config_files=None): - """Construct a ConfigOpts object. + def __init__(self): + """Construct a ConfigOpts object.""" + self._opts = {} # dict of dicts of (opt:, override:, default:) + self._groups = {} - Automatically registers the --config-file option with either a supplied - list of default config files, or a list from find_config_files(). + self._args = None + self._oparser = None + self._cparser = None + self._cli_values = {} + self.__cache = {} + self._config_opts = [] + self._disable_interspersed_args = False - :param project: the toplevel project name, used to locate config files - :param prog: the name of the program (defaults to sys.argv[0] basename) - :param version: the program version (for --version) - :param usage: a usage string (%prog will be expanded) - :param default_config_files: config files to use by default - """ + def _setup(self, project, prog, version, usage, default_config_files): + """Initialize a ConfigOpts object for option parsing.""" if prog is None: prog = os.path.basename(sys.argv[0]) if default_config_files is None: default_config_files = find_config_files(project, prog) + self._oparser = optparse.OptionParser(prog=prog, + version=version, + usage=usage) + if self._disable_interspersed_args: + self._oparser.disable_interspersed_args() + + self._config_opts = [ + MultiStrOpt('config-file', + default=default_config_files, + metavar='PATH', + help='Path to a config file to use. Multiple config ' + 'files can be specified, with values in later ' + 'files taking precedence. The default files ' + ' used are: %s' % (default_config_files, )), + StrOpt('config-dir', + metavar='DIR', + help='Path to a config directory to pull *.conf ' + 'files from. This file set is sorted, so as to ' + 'provide a predictable parse order if individual ' + 'options are over-ridden. The set is parsed after ' + 'the file(s), if any, specified via --config-file, ' + 'hence over-ridden options in the directory take ' + 'precedence.'), + ] + self.register_cli_opts(self._config_opts) + self.project = project self.prog = prog self.version = version self.usage = usage self.default_config_files = default_config_files - self._opts = {} # dict of dicts of (opt:, override:, default:) - self._groups = {} - - self._args = None - self._cli_values = {} - - self._oparser = optparse.OptionParser(prog=self.prog, - version=self.version, - usage=self.usage) - self._cparser = None - - self.__cache = {} - - self.register_cli_opt( - MultiStrOpt('config-file', - default=self.default_config_files, - metavar='PATH', - help='Path to a config file to use. Multiple config ' - 'files can be specified, with values in later ' - 'files taking precedence. The default files used ' - 'are: %s' % (self.default_config_files, ))) - def __clear_cache(f): @functools.wraps(f) def __inner(self, *args, **kwargs): @@ -843,7 +939,13 @@ class ConfigOpts(collections.Mapping): return __inner - def __call__(self, args=None): + def __call__(self, + args=None, + project=None, + prog=None, + version=None, + usage=None, + default_config_files=None): """Parse command line arguments and config files. Calling a ConfigOpts object causes the supplied command line arguments @@ -853,22 +955,34 @@ class ConfigOpts(collections.Mapping): The object may be called multiple times, each time causing the previous set of values to be overwritten. - :params args: command line arguments (defaults to sys.argv[1:]) + Automatically registers the --config-file option with either a supplied + list of default config files, or a list from find_config_files(). + + If the --config-dir option is set, any *.conf files from this + directory are pulled in, after all the file(s) specified by the + --config-file option. + + :param args: command line arguments (defaults to sys.argv[1:]) + :param project: the toplevel project name, used to locate config files + :param prog: the name of the program (defaults to sys.argv[0] basename) + :param version: the program version (for --version) + :param usage: a usage string (%prog will be expanded) + :param default_config_files: config files to use by default :returns: the list of arguments left over after parsing options - :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError + :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError, + RequiredOptError, DuplicateOptError """ - self.reset() + self.clear() - self._args = args + self._setup(project, prog, version, usage, default_config_files) - (values, args) = self._oparser.parse_args(self._args) + self._cli_values, leftovers = self._parse_cli_opts(args) - self._cli_values = vars(values) + self._parse_config_files() - if self.config_file: - self._parse_config_files(self.config_file) + self._check_required_opts() - return args + return leftovers def __getattr__(self, name): """Look up an option value and perform string substitution. @@ -896,12 +1010,21 @@ class ConfigOpts(collections.Mapping): """Return the number of options and option groups.""" return len(self._opts) + len(self._groups) - @__clear_cache def reset(self): - """Reset the state of the object to before it was called.""" + """Clear the object state and unset overrides and defaults.""" + self._unset_defaults_and_overrides() + self.clear() + + @__clear_cache + def clear(self): + """Clear the state of the object to before it was called.""" self._args = None - self._cli_values = None + self._cli_values.clear() + self._oparser = None self._cparser = None + self.unregister_opts(self._config_opts) + for group in self._groups.values(): + group._clear() @__clear_cache def register_opt(self, opt, group=None): @@ -917,7 +1040,7 @@ class ConfigOpts(collections.Mapping): :raises: DuplicateOptError """ if group is not None: - return self._get_group(group)._register_opt(opt) + return self._get_group(group, autocreate=True)._register_opt(opt) if _is_opt_registered(self._opts, opt): return False @@ -948,15 +1071,7 @@ class ConfigOpts(collections.Mapping): if self._args is not None: raise ArgsAlreadyParsedError("cannot register CLI option") - if not self.register_opt(opt, group, clear_cache=False): - return False - - if group is not None: - group = self._get_group(group) - - opt._add_to_cli(self._oparser, group) - - return True + return self.register_opt(opt, group, clear_cache=False) @__clear_cache def register_cli_opts(self, opts, group=None): @@ -977,6 +1092,28 @@ class ConfigOpts(collections.Mapping): self._groups[group.name] = copy.copy(group) + @__clear_cache + def unregister_opt(self, opt, group=None): + """Unregister an option. + + :param opt: an Opt object + :param group: an optional OptGroup object or group name + :raises: ArgsAlreadyParsedError, NoSuchGroupError + """ + if self._args is not None: + raise ArgsAlreadyParsedError("reset before unregistering options") + + if group is not None: + self._get_group(group)._unregister_opt(opt) + elif opt.dest in self._opts: + del self._opts[opt.dest] + + @__clear_cache + def unregister_opts(self, opts, group=None): + """Unregister multiple CLI option schemas at once.""" + for opt in opts: + self.unregister_opt(opt, group, clear_cache=False) + @__clear_cache def set_override(self, name, override, group=None): """Override an opt value. @@ -1007,6 +1144,25 @@ class ConfigOpts(collections.Mapping): opt_info = self._get_opt_info(name, group) opt_info['default'] = default + def _all_opt_infos(self): + """A generator function for iteration opt infos.""" + for info in self._opts.values(): + yield info, None + for group in self._groups.values(): + for info in group._opts.values(): + yield info, group + + def _all_opts(self): + """A generator function for iteration opts.""" + for info, group in self._all_opt_infos(): + yield info['opt'], group + + def _unset_defaults_and_overrides(self): + """Unset any default or override on all options.""" + for info, group in self._all_opt_infos(): + info['default'] = None + info['override'] = None + def disable_interspersed_args(self): """Set parsing to stop on the first non-option. @@ -1024,13 +1180,41 @@ class ConfigOpts(collections.Mapping): i.e. argument parsing is stopped at the first non-option argument. """ - self._oparser.disable_interspersed_args() + self._disable_interspersed_args = True def enable_interspersed_args(self): """Set parsing to not stop on the first non-option. This it the default behaviour.""" - self._oparser.enable_interspersed_args() + self._disable_interspersed_args = False + + def find_file(self, name): + """Locate a file located alongside the config files. + + Search for a file with the supplied basename in the directories + which we have already loaded config files from and other known + configuration directories. + + The directory, if any, supplied by the config_dir option is + searched first. Then the config_file option is iterated over + and each of the base directories of the config_files values + are searched. Failing both of these, the standard directories + searched by the module level find_config_files() function is + used. The first matching file is returned. + + :param basename: the filename, e.g. 'policy.json' + :returns: the path to a matching file, or None + """ + dirs = [] + if self.config_dir: + dirs.append(self.config_dir) + + for cf in reversed(self.config_file): + dirs.append(os.path.dirname(cf)) + + dirs.extend(_get_config_dirs(self.project)) + + return _search_dirs(dirs, name) def log_opt_values(self, logger, lvl): """Log the value of all registered opts. @@ -1100,7 +1284,7 @@ class ConfigOpts(collections.Mapping): return self.GroupAttr(self, self._get_group(name)) info = self._get_opt_info(name, group) - default, opt, override = map(lambda k: info[k], sorted(info.keys())) + default, opt, override = [info[k] for k in sorted(info.keys())] if override is not None: return override @@ -1153,7 +1337,7 @@ class ConfigOpts(collections.Mapping): else: return value - def _get_group(self, group_or_name): + def _get_group(self, group_or_name, autocreate=False): """Looks up a OptGroup object. Helper function to return an OptGroup given a parameter which can @@ -1164,15 +1348,17 @@ class ConfigOpts(collections.Mapping): the API have access to. :param group_or_name: the group's name or the OptGroup object itself + :param autocreate: whether to auto-create the group if it's not found :raises: NoSuchGroupError """ - if isinstance(group_or_name, OptGroup): - group_name = group_or_name.name - else: - group_name = group_or_name + group = group_or_name if isinstance(group_or_name, OptGroup) else None + group_name = group.name if group else group_or_name if not group_name in self._groups: - raise NoSuchGroupError(group_name) + if not group is None or not autocreate: + raise NoSuchGroupError(group_name) + + self.register_group(OptGroup(name=group_name)) return self._groups[group_name] @@ -1194,11 +1380,17 @@ class ConfigOpts(collections.Mapping): return opts[opt_name] - def _parse_config_files(self, config_files): - """Parse the supplied configuration files. + def _parse_config_files(self): + """Parse the config files from --config-file and --config-dir. :raises: ConfigFilesNotFoundError, ConfigFileParseError """ + config_files = list(self.config_file) + + if self.config_dir: + config_dir_glob = os.path.join(self.config_dir, '*.conf') + config_files += sorted(glob.glob(config_dir_glob)) + self._cparser = MultiConfigParser() try: @@ -1210,6 +1402,42 @@ class ConfigOpts(collections.Mapping): not_read_ok = filter(lambda f: f not in read_ok, config_files) raise ConfigFilesNotFoundError(not_read_ok) + def _check_required_opts(self): + """Check that all opts marked as required have values specified. + + :raises: RequiredOptError + """ + for info, group in self._all_opt_infos(): + default, opt, override = [info[k] for k in sorted(info.keys())] + + if opt.required: + if (default is not None or + override is not None): + continue + + if self._get(opt.name, group) is None: + raise RequiredOptError(opt.name, group) + + def _parse_cli_opts(self, args): + """Parse command line options. + + Initializes the command line option parser and parses the supplied + command line arguments. + + :param args: the command line arguments + :returns: a dict of parsed option values + :raises: SystemExit, DuplicateOptError + + """ + self._args = args + + for opt, group in self._all_opts(): + opt._add_to_cli(self._oparser, group) + + values, leftovers = self._oparser.parse_args(args) + + return vars(values), leftovers + class GroupAttr(collections.Mapping): """ @@ -1324,7 +1552,10 @@ class CommonConfigOpts(ConfigOpts): help='syslog facility to receive log lines') ] - def __init__(self, **kwargs): - super(CommonConfigOpts, self).__init__(**kwargs) + def __init__(self): + super(CommonConfigOpts, self).__init__() self.register_cli_opts(self.common_cli_opts) self.register_cli_opts(self.logging_cli_opts) + + +CONF = CommonConfigOpts() diff --git a/heat/openstack/common/importutils.py b/heat/openstack/common/importutils.py index 0561c4df1f..7654af5b95 100644 --- a/heat/openstack/common/importutils.py +++ b/heat/openstack/common/importutils.py @@ -21,8 +21,6 @@ Import related utilities and helper functions. import sys -from heat.openstack.common import exception - def import_class(import_str): """Returns a class from a string including module and class""" @@ -30,8 +28,9 @@ def import_class(import_str): try: __import__(mod_str) return getattr(sys.modules[mod_str], class_str) - except (ImportError, ValueError, AttributeError): - raise exception.NotFound('Class %s cannot be found' % class_str) + except (ImportError, ValueError, AttributeError), exc: + raise ImportError('Class %s cannot be found (%s)' % + (class_str, str(exc))) def import_object(import_str, *args, **kwargs): diff --git a/heat/openstack/common/setup.py b/heat/openstack/common/setup.py index f782b409fd..79b5a62bca 100644 --- a/heat/openstack/common/setup.py +++ b/heat/openstack/common/setup.py @@ -34,7 +34,7 @@ def parse_mailmap(mailmap='.mailmap'): l = l.strip() if not l.startswith('#') and ' ' in l: canonical_email, alias = [x for x in l.split(' ') - if x.startswith('<')] + if x.startswith('<')] mapping[alias] = canonical_email return mapping @@ -61,9 +61,19 @@ def parse_requirements(requirements_files=['requirements.txt', 'tools/pip-requires']): requirements = [] for line in get_reqs_from_files(requirements_files): + # For the requirements list, we need to inject only the portion + # after egg= so that distutils knows the package it's looking for + # such as: + # -e git://github.com/openstack/nova/master#egg=nova if re.match(r'\s*-e\s+', line): requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', line)) + # such as: + # http://github.com/openstack/nova/zipball/master#egg=nova + elif re.match(r'\s*https?:', line): + requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', + line)) + # -f lines are for index locations, and don't get used here elif re.match(r'\s*-f\s+', line): pass else: @@ -75,11 +85,18 @@ def parse_requirements(requirements_files=['requirements.txt', def parse_dependency_links(requirements_files=['requirements.txt', 'tools/pip-requires']): dependency_links = [] + # dependency_links inject alternate locations to find packages listed + # in requirements for line in get_reqs_from_files(requirements_files): + # skip comments and blank lines if re.match(r'(\s*#)|(\s*$)', line): continue + # lines with -e or -f need the whole line, minus the flag if re.match(r'\s*-[ef]\s+', line): dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line)) + # lines that are only urls can go in unmolested + elif re.match(r'\s*https?:', line): + dependency_links.append(line) return dependency_links @@ -137,8 +154,8 @@ def generate_authors(): new_authors = 'AUTHORS' if os.path.isdir('.git'): # don't include jenkins email address in AUTHORS file - git_log_cmd = "git log --format='%aN <%aE>' | sort -u | " \ - "grep -v " + jenkins_email + git_log_cmd = ("git log --format='%aN <%aE>' | sort -u | " + "grep -v " + jenkins_email) changelog = _run_shell_command(git_log_cmd) mailmap = parse_mailmap() with open(new_authors, 'w') as new_authors_fh: diff --git a/heat/openstack/common/timeutils.py b/heat/openstack/common/timeutils.py index c536cb24a4..345f315fbe 100644 --- a/heat/openstack/common/timeutils.py +++ b/heat/openstack/common/timeutils.py @@ -30,7 +30,7 @@ TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" def isotime(at=None): """Stringify time in ISO 8601 format""" if not at: - at = datetime.datetime.utcnow() + at = utcnow() str = at.strftime(TIME_FORMAT) tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' str += ('Z' if tz == 'UTC' else tz)