Update openstack-common

Change-Id: I5af06e0d44a69b9f968fce91db441157a69ea9c7
Signed-off-by: Angus Salkeld <asalkeld@redhat.com>
This commit is contained in:
Angus Salkeld 2012-06-06 15:15:15 +10:00
parent 7e81a8a8f9
commit a724259609
4 changed files with 353 additions and 106 deletions

View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4 # vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 Red Hat, Inc. # Copyright 2012 Red Hat, Inc.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain # not use this file except in compliance with the License. You may obtain
@ -90,16 +90,21 @@ the purposes of --help and CLI arg validation)::
def add_common_opts(conf): def add_common_opts(conf):
conf.register_cli_opts(cli_opts) conf.register_cli_opts(cli_opts)
The config manager has a single CLI option defined by default, --config-file:: The config manager has two CLI options defined by default, --config-file
and --config-dir::
class ConfigOpts(object): class ConfigOpts(object):
config_file_opt = MultiStrOpt('config-file', def __call__(self, ...):
...
def __init__(self, ...): opts = [
... MultiStrOpt('config-file',
self.register_cli_opt(self.config_file_opt) ...),
StrOpt('config-dir',
...),
]
self.register_cli_opts(opts)
Option values are parsed from any supplied config files using Option values are parsed from any supplied config files using
openstack.common.iniparser. If none are specified, a default set is used openstack.common.iniparser. If none are specified, a default set is used
@ -144,6 +149,14 @@ Options can be registered as belonging to a group::
conf.register_opt(rabbit_host_opt, group=rabbit_group) conf.register_opt(rabbit_host_opt, group=rabbit_group)
conf.register_opt(rabbit_port_opt, group='rabbit') conf.register_opt(rabbit_port_opt, group='rabbit')
If it no group attributes are required other than the group name, the group
need not be explicitly registered e.g.
def register_rabbit_opts(conf):
# The group will automatically be created, equivalent calling::
# conf.register_group(OptGroup(name='rabbit'))
conf.register_opt(rabbit_port_opt, group='rabbit')
If no group is specified, options belong to the 'DEFAULT' section of config If no group is specified, options belong to the 'DEFAULT' section of config
files:: files::
@ -208,6 +221,9 @@ as the leftover arguments, but will instead return::
i.e. argument parsing is stopped at the first non-option argument. i.e. argument parsing is stopped at the first non-option argument.
Options may be declared as required so that an error is raised if the user
does not supply a value for the option.
Options may be declared as secret so that their values are not leaked into Options may be declared as secret so that their values are not leaked into
log files: log files:
@ -217,11 +233,28 @@ log files:
... ...
] ]
This module also contains a global instance of the CommonConfigOpts class
in order to support a common usage pattern in OpenStack:
from openstack.common import cfg
opts = [
cfg.StrOpt('bind_host' default='0.0.0.0'),
cfg.IntOpt('bind_port', default=9292),
]
CONF = cfg.CONF
CONF.register_opts(opts)
def start(server, app):
server.start(app, CONF.bind_port, CONF.bind_host)
""" """
import collections import collections
import copy import copy
import functools import functools
import glob
import optparse import optparse
import os import os
import string import string
@ -285,6 +318,21 @@ class DuplicateOptError(Error):
return "duplicate option: %s" % self.opt_name return "duplicate option: %s" % self.opt_name
class RequiredOptError(Error):
"""Raised if an option is required but no value is supplied by the user."""
def __init__(self, opt_name, group=None):
self.opt_name = opt_name
self.group = group
def __str__(self):
if self.group is None:
return "value required for option: %s" % self.opt_name
else:
return "value required for option: %s.%s" % (self.group.name,
self.opt_name)
class TemplateSubstitutionError(Error): class TemplateSubstitutionError(Error):
"""Raised if an error occurs substituting a variable in an opt value.""" """Raised if an error occurs substituting a variable in an opt value."""
@ -319,6 +367,52 @@ class ConfigFileValueError(Error):
pass pass
def _get_config_dirs(project=None):
"""Return a list of directors where config files may be located.
:param project: an optional project name
If a project is specified, following directories are returned::
~/.${project}/
~/
/etc/${project}/
/etc/
Otherwise, these directories::
~/
/etc/
"""
fix_path = lambda p: os.path.abspath(os.path.expanduser(p))
cfg_dirs = [
fix_path(os.path.join('~', '.' + project)) if project else None,
fix_path('~'),
os.path.join('/etc', project) if project else None,
'/etc'
]
return filter(bool, cfg_dirs)
def _search_dirs(dirs, basename, extension=""):
"""Search a list of directories for a given filename.
Iterator over the supplied directories, returning the first file
found with the supplied name and extension.
:param dirs: a list of directories
:param basename: the filename, e.g. 'glance-api'
:param extension: the file extension, e.g. '.conf'
:returns: the path to a matching file, or None
"""
for d in dirs:
path = os.path.join(d, '%s%s' % (basename, extension))
if os.path.exists(path):
return path
def find_config_files(project=None, prog=None, extension='.conf'): def find_config_files(project=None, prog=None, extension='.conf'):
"""Return a list of default configuration files. """Return a list of default configuration files.
@ -347,26 +441,12 @@ def find_config_files(project=None, prog=None, extension='.conf'):
if prog is None: if prog is None:
prog = os.path.basename(sys.argv[0]) prog = os.path.basename(sys.argv[0])
fix_path = lambda p: os.path.abspath(os.path.expanduser(p)) cfg_dirs = _get_config_dirs(project)
cfg_dirs = [
fix_path(os.path.join('~', '.' + project)) if project else None,
fix_path('~'),
os.path.join('/etc', project) if project else None,
'/etc'
]
cfg_dirs = filter(bool, cfg_dirs)
def search_dirs(dirs, basename, extension):
for d in dirs:
path = os.path.join(d, '%s%s' % (basename, extension))
if os.path.exists(path):
return path
config_files = [] config_files = []
if project: if project:
config_files.append(search_dirs(cfg_dirs, project, extension)) config_files.append(_search_dirs(cfg_dirs, project, extension))
config_files.append(search_dirs(cfg_dirs, prog, extension)) config_files.append(_search_dirs(cfg_dirs, prog, extension))
return filter(bool, config_files) return filter(bool, config_files)
@ -414,7 +494,7 @@ class Opt(object):
multi = False multi = False
def __init__(self, name, dest=None, short=None, default=None, def __init__(self, name, dest=None, short=None, default=None,
metavar=None, help=None, secret=False): metavar=None, help=None, secret=False, required=False):
"""Construct an Opt object. """Construct an Opt object.
The only required parameter is the option's name. However, it is The only required parameter is the option's name. However, it is
@ -427,6 +507,7 @@ class Opt(object):
:param metavar: the option argument to show in --help :param metavar: the option argument to show in --help
:param help: an explanation of how the option is used :param help: an explanation of how the option is used
:param secret: true iff the value should be obfuscated in log output :param secret: true iff the value should be obfuscated in log output
:param required: true iff a value must be supplied for this option
""" """
self.name = name self.name = name
if dest is None: if dest is None:
@ -438,6 +519,7 @@ class Opt(object):
self.metavar = metavar self.metavar = metavar
self.help = help self.help = help
self.secret = secret self.secret = secret
self.required = required
def _get_from_config_parser(self, cparser, section): def _get_from_config_parser(self, cparser, section):
"""Retrieves the option value from a MultiConfigParser object. """Retrieves the option value from a MultiConfigParser object.
@ -702,6 +784,14 @@ class OptGroup(object):
return True return True
def _unregister_opt(self, opt):
"""Remove an opt from this group.
:param opt: an Opt object
"""
if opt.dest in self._opts:
del self._opts[opt.dest]
def _get_optparse_group(self, parser): def _get_optparse_group(self, parser):
"""Build an optparse.OptionGroup for this group.""" """Build an optparse.OptionGroup for this group."""
if self._optparse_group is None: if self._optparse_group is None:
@ -709,6 +799,10 @@ class OptGroup(object):
self.help) self.help)
return self._optparse_group return self._optparse_group
def _clear(self):
"""Clear this group's option parsing state."""
self._optparse_group = None
class ParseError(iniparser.ParseError): class ParseError(iniparser.ParseError):
def __init__(self, msg, lineno, line, filename): def __init__(self, msg, lineno, line, filename):
@ -783,57 +877,59 @@ class ConfigOpts(collections.Mapping):
the values of options. the values of options.
""" """
def __init__(self, def __init__(self):
project=None, """Construct a ConfigOpts object."""
prog=None, self._opts = {} # dict of dicts of (opt:, override:, default:)
version=None, self._groups = {}
usage=None,
default_config_files=None):
"""Construct a ConfigOpts object.
Automatically registers the --config-file option with either a supplied self._args = None
list of default config files, or a list from find_config_files(). self._oparser = None
self._cparser = None
self._cli_values = {}
self.__cache = {}
self._config_opts = []
self._disable_interspersed_args = False
:param project: the toplevel project name, used to locate config files def _setup(self, project, prog, version, usage, default_config_files):
:param prog: the name of the program (defaults to sys.argv[0] basename) """Initialize a ConfigOpts object for option parsing."""
:param version: the program version (for --version)
:param usage: a usage string (%prog will be expanded)
:param default_config_files: config files to use by default
"""
if prog is None: if prog is None:
prog = os.path.basename(sys.argv[0]) prog = os.path.basename(sys.argv[0])
if default_config_files is None: if default_config_files is None:
default_config_files = find_config_files(project, prog) default_config_files = find_config_files(project, prog)
self._oparser = optparse.OptionParser(prog=prog,
version=version,
usage=usage)
if self._disable_interspersed_args:
self._oparser.disable_interspersed_args()
self._config_opts = [
MultiStrOpt('config-file',
default=default_config_files,
metavar='PATH',
help='Path to a config file to use. Multiple config '
'files can be specified, with values in later '
'files taking precedence. The default files '
' used are: %s' % (default_config_files, )),
StrOpt('config-dir',
metavar='DIR',
help='Path to a config directory to pull *.conf '
'files from. This file set is sorted, so as to '
'provide a predictable parse order if individual '
'options are over-ridden. The set is parsed after '
'the file(s), if any, specified via --config-file, '
'hence over-ridden options in the directory take '
'precedence.'),
]
self.register_cli_opts(self._config_opts)
self.project = project self.project = project
self.prog = prog self.prog = prog
self.version = version self.version = version
self.usage = usage self.usage = usage
self.default_config_files = default_config_files self.default_config_files = default_config_files
self._opts = {} # dict of dicts of (opt:, override:, default:)
self._groups = {}
self._args = None
self._cli_values = {}
self._oparser = optparse.OptionParser(prog=self.prog,
version=self.version,
usage=self.usage)
self._cparser = None
self.__cache = {}
self.register_cli_opt(
MultiStrOpt('config-file',
default=self.default_config_files,
metavar='PATH',
help='Path to a config file to use. Multiple config '
'files can be specified, with values in later '
'files taking precedence. The default files used '
'are: %s' % (self.default_config_files, )))
def __clear_cache(f): def __clear_cache(f):
@functools.wraps(f) @functools.wraps(f)
def __inner(self, *args, **kwargs): def __inner(self, *args, **kwargs):
@ -843,7 +939,13 @@ class ConfigOpts(collections.Mapping):
return __inner return __inner
def __call__(self, args=None): def __call__(self,
args=None,
project=None,
prog=None,
version=None,
usage=None,
default_config_files=None):
"""Parse command line arguments and config files. """Parse command line arguments and config files.
Calling a ConfigOpts object causes the supplied command line arguments Calling a ConfigOpts object causes the supplied command line arguments
@ -853,22 +955,34 @@ class ConfigOpts(collections.Mapping):
The object may be called multiple times, each time causing the previous The object may be called multiple times, each time causing the previous
set of values to be overwritten. set of values to be overwritten.
:params args: command line arguments (defaults to sys.argv[1:]) Automatically registers the --config-file option with either a supplied
list of default config files, or a list from find_config_files().
If the --config-dir option is set, any *.conf files from this
directory are pulled in, after all the file(s) specified by the
--config-file option.
:param args: command line arguments (defaults to sys.argv[1:])
:param project: the toplevel project name, used to locate config files
:param prog: the name of the program (defaults to sys.argv[0] basename)
:param version: the program version (for --version)
:param usage: a usage string (%prog will be expanded)
:param default_config_files: config files to use by default
:returns: the list of arguments left over after parsing options :returns: the list of arguments left over after parsing options
:raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError,
RequiredOptError, DuplicateOptError
""" """
self.reset() self.clear()
self._args = args self._setup(project, prog, version, usage, default_config_files)
(values, args) = self._oparser.parse_args(self._args) self._cli_values, leftovers = self._parse_cli_opts(args)
self._cli_values = vars(values) self._parse_config_files()
if self.config_file: self._check_required_opts()
self._parse_config_files(self.config_file)
return args return leftovers
def __getattr__(self, name): def __getattr__(self, name):
"""Look up an option value and perform string substitution. """Look up an option value and perform string substitution.
@ -896,12 +1010,21 @@ class ConfigOpts(collections.Mapping):
"""Return the number of options and option groups.""" """Return the number of options and option groups."""
return len(self._opts) + len(self._groups) return len(self._opts) + len(self._groups)
@__clear_cache
def reset(self): def reset(self):
"""Reset the state of the object to before it was called.""" """Clear the object state and unset overrides and defaults."""
self._unset_defaults_and_overrides()
self.clear()
@__clear_cache
def clear(self):
"""Clear the state of the object to before it was called."""
self._args = None self._args = None
self._cli_values = None self._cli_values.clear()
self._oparser = None
self._cparser = None self._cparser = None
self.unregister_opts(self._config_opts)
for group in self._groups.values():
group._clear()
@__clear_cache @__clear_cache
def register_opt(self, opt, group=None): def register_opt(self, opt, group=None):
@ -917,7 +1040,7 @@ class ConfigOpts(collections.Mapping):
:raises: DuplicateOptError :raises: DuplicateOptError
""" """
if group is not None: if group is not None:
return self._get_group(group)._register_opt(opt) return self._get_group(group, autocreate=True)._register_opt(opt)
if _is_opt_registered(self._opts, opt): if _is_opt_registered(self._opts, opt):
return False return False
@ -948,15 +1071,7 @@ class ConfigOpts(collections.Mapping):
if self._args is not None: if self._args is not None:
raise ArgsAlreadyParsedError("cannot register CLI option") raise ArgsAlreadyParsedError("cannot register CLI option")
if not self.register_opt(opt, group, clear_cache=False): return self.register_opt(opt, group, clear_cache=False)
return False
if group is not None:
group = self._get_group(group)
opt._add_to_cli(self._oparser, group)
return True
@__clear_cache @__clear_cache
def register_cli_opts(self, opts, group=None): def register_cli_opts(self, opts, group=None):
@ -977,6 +1092,28 @@ class ConfigOpts(collections.Mapping):
self._groups[group.name] = copy.copy(group) self._groups[group.name] = copy.copy(group)
@__clear_cache
def unregister_opt(self, opt, group=None):
"""Unregister an option.
:param opt: an Opt object
:param group: an optional OptGroup object or group name
:raises: ArgsAlreadyParsedError, NoSuchGroupError
"""
if self._args is not None:
raise ArgsAlreadyParsedError("reset before unregistering options")
if group is not None:
self._get_group(group)._unregister_opt(opt)
elif opt.dest in self._opts:
del self._opts[opt.dest]
@__clear_cache
def unregister_opts(self, opts, group=None):
"""Unregister multiple CLI option schemas at once."""
for opt in opts:
self.unregister_opt(opt, group, clear_cache=False)
@__clear_cache @__clear_cache
def set_override(self, name, override, group=None): def set_override(self, name, override, group=None):
"""Override an opt value. """Override an opt value.
@ -1007,6 +1144,25 @@ class ConfigOpts(collections.Mapping):
opt_info = self._get_opt_info(name, group) opt_info = self._get_opt_info(name, group)
opt_info['default'] = default opt_info['default'] = default
def _all_opt_infos(self):
"""A generator function for iteration opt infos."""
for info in self._opts.values():
yield info, None
for group in self._groups.values():
for info in group._opts.values():
yield info, group
def _all_opts(self):
"""A generator function for iteration opts."""
for info, group in self._all_opt_infos():
yield info['opt'], group
def _unset_defaults_and_overrides(self):
"""Unset any default or override on all options."""
for info, group in self._all_opt_infos():
info['default'] = None
info['override'] = None
def disable_interspersed_args(self): def disable_interspersed_args(self):
"""Set parsing to stop on the first non-option. """Set parsing to stop on the first non-option.
@ -1024,13 +1180,41 @@ class ConfigOpts(collections.Mapping):
i.e. argument parsing is stopped at the first non-option argument. i.e. argument parsing is stopped at the first non-option argument.
""" """
self._oparser.disable_interspersed_args() self._disable_interspersed_args = True
def enable_interspersed_args(self): def enable_interspersed_args(self):
"""Set parsing to not stop on the first non-option. """Set parsing to not stop on the first non-option.
This it the default behaviour.""" This it the default behaviour."""
self._oparser.enable_interspersed_args() self._disable_interspersed_args = False
def find_file(self, name):
"""Locate a file located alongside the config files.
Search for a file with the supplied basename in the directories
which we have already loaded config files from and other known
configuration directories.
The directory, if any, supplied by the config_dir option is
searched first. Then the config_file option is iterated over
and each of the base directories of the config_files values
are searched. Failing both of these, the standard directories
searched by the module level find_config_files() function is
used. The first matching file is returned.
:param basename: the filename, e.g. 'policy.json'
:returns: the path to a matching file, or None
"""
dirs = []
if self.config_dir:
dirs.append(self.config_dir)
for cf in reversed(self.config_file):
dirs.append(os.path.dirname(cf))
dirs.extend(_get_config_dirs(self.project))
return _search_dirs(dirs, name)
def log_opt_values(self, logger, lvl): def log_opt_values(self, logger, lvl):
"""Log the value of all registered opts. """Log the value of all registered opts.
@ -1100,7 +1284,7 @@ class ConfigOpts(collections.Mapping):
return self.GroupAttr(self, self._get_group(name)) return self.GroupAttr(self, self._get_group(name))
info = self._get_opt_info(name, group) info = self._get_opt_info(name, group)
default, opt, override = map(lambda k: info[k], sorted(info.keys())) default, opt, override = [info[k] for k in sorted(info.keys())]
if override is not None: if override is not None:
return override return override
@ -1153,7 +1337,7 @@ class ConfigOpts(collections.Mapping):
else: else:
return value return value
def _get_group(self, group_or_name): def _get_group(self, group_or_name, autocreate=False):
"""Looks up a OptGroup object. """Looks up a OptGroup object.
Helper function to return an OptGroup given a parameter which can Helper function to return an OptGroup given a parameter which can
@ -1164,15 +1348,17 @@ class ConfigOpts(collections.Mapping):
the API have access to. the API have access to.
:param group_or_name: the group's name or the OptGroup object itself :param group_or_name: the group's name or the OptGroup object itself
:param autocreate: whether to auto-create the group if it's not found
:raises: NoSuchGroupError :raises: NoSuchGroupError
""" """
if isinstance(group_or_name, OptGroup): group = group_or_name if isinstance(group_or_name, OptGroup) else None
group_name = group_or_name.name group_name = group.name if group else group_or_name
else:
group_name = group_or_name
if not group_name in self._groups: if not group_name in self._groups:
raise NoSuchGroupError(group_name) if not group is None or not autocreate:
raise NoSuchGroupError(group_name)
self.register_group(OptGroup(name=group_name))
return self._groups[group_name] return self._groups[group_name]
@ -1194,11 +1380,17 @@ class ConfigOpts(collections.Mapping):
return opts[opt_name] return opts[opt_name]
def _parse_config_files(self, config_files): def _parse_config_files(self):
"""Parse the supplied configuration files. """Parse the config files from --config-file and --config-dir.
:raises: ConfigFilesNotFoundError, ConfigFileParseError :raises: ConfigFilesNotFoundError, ConfigFileParseError
""" """
config_files = list(self.config_file)
if self.config_dir:
config_dir_glob = os.path.join(self.config_dir, '*.conf')
config_files += sorted(glob.glob(config_dir_glob))
self._cparser = MultiConfigParser() self._cparser = MultiConfigParser()
try: try:
@ -1210,6 +1402,42 @@ class ConfigOpts(collections.Mapping):
not_read_ok = filter(lambda f: f not in read_ok, config_files) not_read_ok = filter(lambda f: f not in read_ok, config_files)
raise ConfigFilesNotFoundError(not_read_ok) raise ConfigFilesNotFoundError(not_read_ok)
def _check_required_opts(self):
"""Check that all opts marked as required have values specified.
:raises: RequiredOptError
"""
for info, group in self._all_opt_infos():
default, opt, override = [info[k] for k in sorted(info.keys())]
if opt.required:
if (default is not None or
override is not None):
continue
if self._get(opt.name, group) is None:
raise RequiredOptError(opt.name, group)
def _parse_cli_opts(self, args):
"""Parse command line options.
Initializes the command line option parser and parses the supplied
command line arguments.
:param args: the command line arguments
:returns: a dict of parsed option values
:raises: SystemExit, DuplicateOptError
"""
self._args = args
for opt, group in self._all_opts():
opt._add_to_cli(self._oparser, group)
values, leftovers = self._oparser.parse_args(args)
return vars(values), leftovers
class GroupAttr(collections.Mapping): class GroupAttr(collections.Mapping):
""" """
@ -1324,7 +1552,10 @@ class CommonConfigOpts(ConfigOpts):
help='syslog facility to receive log lines') help='syslog facility to receive log lines')
] ]
def __init__(self, **kwargs): def __init__(self):
super(CommonConfigOpts, self).__init__(**kwargs) super(CommonConfigOpts, self).__init__()
self.register_cli_opts(self.common_cli_opts) self.register_cli_opts(self.common_cli_opts)
self.register_cli_opts(self.logging_cli_opts) self.register_cli_opts(self.logging_cli_opts)
CONF = CommonConfigOpts()

View File

@ -21,8 +21,6 @@ Import related utilities and helper functions.
import sys import sys
from heat.openstack.common import exception
def import_class(import_str): def import_class(import_str):
"""Returns a class from a string including module and class""" """Returns a class from a string including module and class"""
@ -30,8 +28,9 @@ def import_class(import_str):
try: try:
__import__(mod_str) __import__(mod_str)
return getattr(sys.modules[mod_str], class_str) return getattr(sys.modules[mod_str], class_str)
except (ImportError, ValueError, AttributeError): except (ImportError, ValueError, AttributeError), exc:
raise exception.NotFound('Class %s cannot be found' % class_str) raise ImportError('Class %s cannot be found (%s)' %
(class_str, str(exc)))
def import_object(import_str, *args, **kwargs): def import_object(import_str, *args, **kwargs):

View File

@ -34,7 +34,7 @@ def parse_mailmap(mailmap='.mailmap'):
l = l.strip() l = l.strip()
if not l.startswith('#') and ' ' in l: if not l.startswith('#') and ' ' in l:
canonical_email, alias = [x for x in l.split(' ') canonical_email, alias = [x for x in l.split(' ')
if x.startswith('<')] if x.startswith('<')]
mapping[alias] = canonical_email mapping[alias] = canonical_email
return mapping return mapping
@ -61,9 +61,19 @@ def parse_requirements(requirements_files=['requirements.txt',
'tools/pip-requires']): 'tools/pip-requires']):
requirements = [] requirements = []
for line in get_reqs_from_files(requirements_files): for line in get_reqs_from_files(requirements_files):
# For the requirements list, we need to inject only the portion
# after egg= so that distutils knows the package it's looking for
# such as:
# -e git://github.com/openstack/nova/master#egg=nova
if re.match(r'\s*-e\s+', line): if re.match(r'\s*-e\s+', line):
requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1',
line)) line))
# such as:
# http://github.com/openstack/nova/zipball/master#egg=nova
elif re.match(r'\s*https?:', line):
requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1',
line))
# -f lines are for index locations, and don't get used here
elif re.match(r'\s*-f\s+', line): elif re.match(r'\s*-f\s+', line):
pass pass
else: else:
@ -75,11 +85,18 @@ def parse_requirements(requirements_files=['requirements.txt',
def parse_dependency_links(requirements_files=['requirements.txt', def parse_dependency_links(requirements_files=['requirements.txt',
'tools/pip-requires']): 'tools/pip-requires']):
dependency_links = [] dependency_links = []
# dependency_links inject alternate locations to find packages listed
# in requirements
for line in get_reqs_from_files(requirements_files): for line in get_reqs_from_files(requirements_files):
# skip comments and blank lines
if re.match(r'(\s*#)|(\s*$)', line): if re.match(r'(\s*#)|(\s*$)', line):
continue continue
# lines with -e or -f need the whole line, minus the flag
if re.match(r'\s*-[ef]\s+', line): if re.match(r'\s*-[ef]\s+', line):
dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line)) dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line))
# lines that are only urls can go in unmolested
elif re.match(r'\s*https?:', line):
dependency_links.append(line)
return dependency_links return dependency_links
@ -137,8 +154,8 @@ def generate_authors():
new_authors = 'AUTHORS' new_authors = 'AUTHORS'
if os.path.isdir('.git'): if os.path.isdir('.git'):
# don't include jenkins email address in AUTHORS file # don't include jenkins email address in AUTHORS file
git_log_cmd = "git log --format='%aN <%aE>' | sort -u | " \ git_log_cmd = ("git log --format='%aN <%aE>' | sort -u | "
"grep -v " + jenkins_email "grep -v " + jenkins_email)
changelog = _run_shell_command(git_log_cmd) changelog = _run_shell_command(git_log_cmd)
mailmap = parse_mailmap() mailmap = parse_mailmap()
with open(new_authors, 'w') as new_authors_fh: with open(new_authors, 'w') as new_authors_fh:

View File

@ -30,7 +30,7 @@ TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
def isotime(at=None): def isotime(at=None):
"""Stringify time in ISO 8601 format""" """Stringify time in ISO 8601 format"""
if not at: if not at:
at = datetime.datetime.utcnow() at = utcnow()
str = at.strftime(TIME_FORMAT) str = at.strftime(TIME_FORMAT)
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
str += ('Z' if tz == 'UTC' else tz) str += ('Z' if tz == 'UTC' else tz)