diff --git a/HACKING.rst b/HACKING.rst index c6c39ec466..42e36a3139 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -184,3 +184,18 @@ For every new feature, unit tests should be created that both test and bug that had no unit test, a new passing unit test should be added. If a submitted bug fix does have a unit test, be sure to add a new one that fails without the patch and passes with the patch. + + +openstack-common +---------------- + +A number of modules from openstack-common are imported into the project. + +These modules are "incubating" in openstack-common and are kept in sync +with the help of openstack-common's update.py script. See: + + http://wiki.openstack.org/CommonLibrary#Incubation + +The copy of the code should never be directly modified here. Please +always update openstack-common first and then run the script to copy +the changes across. diff --git a/bin/glance-control b/bin/glance-control index 15473ac373..e35b5d4e4e 100755 --- a/bin/glance-control +++ b/bin/glance-control @@ -43,8 +43,8 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): gettext.install('glance', unicode=1) -from glance.common import cfg from glance.common import config +from glance.openstack.common import cfg ALL_COMMANDS = ['start', 'stop', 'shutdown', 'restart', 'reload', 'force-reload'] diff --git a/bin/glance-manage b/bin/glance-manage index 3a50c115e8..59296fdf19 100755 --- a/bin/glance-manage +++ b/bin/glance-manage @@ -40,9 +40,9 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): gettext.install('glance', unicode=1) -from glance.common import cfg from glance.common import config from glance.common import exception +from glance.openstack.common import cfg import glance.registry.db import glance.registry.db.migration diff --git a/bin/glance-scrubber b/bin/glance-scrubber index 1e9ffe420e..6660239257 100755 --- a/bin/glance-scrubber +++ b/bin/glance-scrubber @@ -34,8 +34,8 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): gettext.install('glance', unicode=1) -from glance.common import cfg from glance.common import config +from glance.openstack.common import cfg from glance.store import scrubber diff --git a/glance/api/policy.py b/glance/api/policy.py index 8a6ff0151c..5fcdacd780 100644 --- a/glance/api/policy.py +++ b/glance/api/policy.py @@ -20,9 +20,9 @@ import json import os.path -from glance.common import cfg from glance.common import exception from glance.common import policy +from glance.openstack.common import cfg class Enforcer(object): @@ -58,7 +58,7 @@ class Enforcer(object): if conf.policy_file: return conf.policy_file - matches = cfg.find_config_files('glance', 'policy', 'json') + matches = cfg.find_config_files('glance', 'policy', '.json') try: return matches[0] diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index 9839d2e2bf..0312e5ce8f 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -37,10 +37,10 @@ from glance.api import policy import glance.api.v1 from glance.api.v1 import controller from glance.api.v1 import filters -from glance.common import cfg from glance.common import exception from glance.common import wsgi from glance.common import utils +from glance.openstack.common import cfg import glance.store import glance.store.filesystem import glance.store.http diff --git a/glance/common/config.py b/glance/common/config.py index da1bcdb39c..b6a08d17e4 100644 --- a/glance/common/config.py +++ b/glance/common/config.py @@ -26,8 +26,8 @@ import logging.handlers import os import sys -from glance.common import cfg from glance.common import wsgi +from glance.openstack.common import cfg from glance import version diff --git a/glance/common/context.py b/glance/common/context.py index 06e815f163..6baaa0b7e3 100644 --- a/glance/common/context.py +++ b/glance/common/context.py @@ -15,10 +15,9 @@ # License for the specific language governing permissions and limitations # under the License. -from glance.common import cfg from glance.common import exception -from glance.common import utils from glance.common import wsgi +from glance.openstack.common import cfg from glance.registry.db import api as db_api diff --git a/glance/common/wsgi.py b/glance/common/wsgi.py index 47eab3b438..9ffecc7c3e 100644 --- a/glance/common/wsgi.py +++ b/glance/common/wsgi.py @@ -40,9 +40,9 @@ import routes.middleware import webob.dec import webob.exc -from glance.common import cfg from glance.common import exception from glance.common import utils +from glance.openstack.common import cfg bind_opts = [ diff --git a/glance/image_cache/__init__.py b/glance/image_cache/__init__.py index 67936dd8e2..8202a32129 100644 --- a/glance/image_cache/__init__.py +++ b/glance/image_cache/__init__.py @@ -21,9 +21,9 @@ LRU Cache for Image Data import logging -from glance.common import cfg from glance.common import exception from glance.common import utils +from glance.openstack.common import cfg logger = logging.getLogger(__name__) DEFAULT_MAX_CACHE_SIZE = 10 * 1024 * 1024 * 1024 # 10 GB diff --git a/glance/image_cache/drivers/sqlite.py b/glance/image_cache/drivers/sqlite.py index b0b8db1071..8606fd778c 100644 --- a/glance/image_cache/drivers/sqlite.py +++ b/glance/image_cache/drivers/sqlite.py @@ -29,9 +29,9 @@ import time from eventlet import sleep, timeout import sqlite3 -from glance.common import cfg from glance.common import exception from glance.image_cache.drivers import base +from glance.openstack.common import cfg logger = logging.getLogger(__name__) DEFAULT_SQL_CALL_TIMEOUT = 2 diff --git a/glance/notifier/__init__.py b/glance/notifier/__init__.py index c3ead3ed34..14efb3c117 100644 --- a/glance/notifier/__init__.py +++ b/glance/notifier/__init__.py @@ -20,9 +20,9 @@ import datetime import socket import uuid -from glance.common import cfg from glance.common import exception from glance.common import utils +from glance.openstack.common import cfg _STRATEGIES = { diff --git a/glance/notifier/notify_kombu.py b/glance/notifier/notify_kombu.py index 6623fdce03..b026abcd12 100644 --- a/glance/notifier/notify_kombu.py +++ b/glance/notifier/notify_kombu.py @@ -21,8 +21,8 @@ import time import kombu.connection import kombu.entity -from glance.common import cfg from glance.notifier import strategy +from glance.openstack.common import cfg logger = logging.getLogger('glance.notifier.notify_kombu') diff --git a/glance/notifier/notify_qpid.py b/glance/notifier/notify_qpid.py index a0535a6bf9..aaf71372b7 100644 --- a/glance/notifier/notify_qpid.py +++ b/glance/notifier/notify_qpid.py @@ -19,8 +19,8 @@ import logging import qpid.messaging -from glance.common import cfg from glance.notifier import strategy +from glance.openstack.common import cfg logger = logging.getLogger('glance.notifier.notify_qpid') diff --git a/glance/openstack/__init__.py b/glance/openstack/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/glance/openstack/common/README b/glance/openstack/common/README new file mode 100644 index 0000000000..def4a172aa --- /dev/null +++ b/glance/openstack/common/README @@ -0,0 +1,13 @@ +openstack-common +---------------- + +A number of modules from openstack-common are imported into this project. + +These modules are "incubating" in openstack-common and are kept in sync +with the help of openstack-common's update.py script. See: + + http://wiki.openstack.org/CommonLibrary#Incubation + +The copy of the code should never be directly modified here. Please +always update openstack-common first and then run the script to copy +the changes across. diff --git a/glance/openstack/common/__init__.py b/glance/openstack/common/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/glance/common/cfg.py b/glance/openstack/common/cfg.py similarity index 77% rename from glance/common/cfg.py rename to glance/openstack/common/cfg.py index 383dec4bf2..1228cdeceb 100644 --- a/glance/common/cfg.py +++ b/glance/openstack/common/cfg.py @@ -17,7 +17,9 @@ r""" Configuration options which may be set on the command line or in config files. -The schema for each option is defined using the Opt sub-classes e.g. +The schema for each option is defined using the Opt sub-classes, e.g.: + +:: common_opts = [ cfg.StrOpt('bind_host', @@ -28,22 +30,20 @@ The schema for each option is defined using the Opt sub-classes e.g. help='Port number to listen on') ] -Options can be strings, integers, floats, booleans, lists or 'multi strings': +Options can be strings, integers, floats, booleans, lists or 'multi strings':: - enabled_apis_opt = \ - cfg.ListOpt('enabled_apis', - default=['ec2', 'osapi'], - help='List of APIs to enable by default') + enabled_apis_opt = cfg.ListOpt('enabled_apis', + default=['ec2', 'osapi_compute'], + help='List of APIs to enable by default') DEFAULT_EXTENSIONS = [ - 'nova.api.openstack.contrib.standard_extensions' + 'nova.api.openstack.compute.contrib.standard_extensions' ] - osapi_extension_opt = \ - cfg.MultiStrOpt('osapi_extension', - default=DEFAULT_EXTENSIONS) + osapi_compute_extension_opt = cfg.MultiStrOpt('osapi_compute_extension', + default=DEFAULT_EXTENSIONS) Option schemas are registered with with the config manager at runtime, but -before the option is referenced: +before the option is referenced:: class ExtensionManager(object): @@ -55,11 +55,11 @@ before the option is referenced: ... def _load_extensions(self): - for ext_factory in self.conf.osapi_extension: + for ext_factory in self.conf.osapi_compute_extension: .... A common usage pattern is for each option schema to be defined in the module or -class which uses the option: +class which uses the option:: opts = ... @@ -74,7 +74,7 @@ class which uses the option: An option may optionally be made available via the command line. Such options must registered with the config manager before the command line is parsed (for -the purposes of --help and CLI arg validation): +the purposes of --help and CLI arg validation):: cli_opts = [ cfg.BoolOpt('verbose', @@ -90,21 +90,20 @@ the purposes of --help and CLI arg validation): def add_common_opts(conf): conf.register_cli_opts(cli_opts) -The config manager has a single CLI option defined by default, --config-file: +The config manager has a single CLI option defined by default, --config-file:: class ConfigOpts(object): - config_file_opt = \ - MultiStrOpt('config-file', - ... + config_file_opt = MultiStrOpt('config-file', + ... def __init__(self, ...): ... self.register_cli_opt(self.config_file_opt) -Option values are parsed from any supplied config files using SafeConfigParser. -If none are specified, a default set is used e.g. glance-api.conf and -glance-common.conf: +Option values are parsed from any supplied config files using +openstack.common.iniparser. If none are specified, a default set is used +e.g. glance-api.conf and glance-common.conf:: glance-api.conf: [DEFAULT] @@ -119,7 +118,7 @@ are parsed in order, with values in later files overriding those in earlier files. The parsing of CLI args and config files is initiated by invoking the config -manager e.g. +manager e.g.:: conf = ConfigOpts() conf.register_opt(BoolOpt('verbose', ...)) @@ -127,34 +126,26 @@ manager e.g. if conf.verbose: ... -Options can be registered as belonging to a group: +Options can be registered as belonging to a group:: - rabbit_group = cfg.OptionGroup(name='rabbit', - title='RabbitMQ options') + rabbit_group = cfg.OptGroup(name='rabbit', + title='RabbitMQ options') - rabbit_host_opt = \ - cfg.StrOpt('host', - group='rabbit', - default='localhost', - help='IP/hostname to listen on'), - rabbit_port_opt = \ - cfg.IntOpt('port', - default=5672, - help='Port number to listen on') - rabbit_ssl_opt = \ - conf.BoolOpt('use_ssl', - default=False, - help='Whether to support SSL connections') + rabbit_host_opt = cfg.StrOpt('host', + default='localhost', + help='IP/hostname to listen on'), + rabbit_port_opt = cfg.IntOpt('port', + default=5672, + help='Port number to listen on') def register_rabbit_opts(conf): conf.register_group(rabbit_group) - # options can be registered under a group in any of these ways: - conf.register_opt(rabbit_host_opt) + # options can be registered under a group in either of these ways: + conf.register_opt(rabbit_host_opt, group=rabbit_group) conf.register_opt(rabbit_port_opt, group='rabbit') - conf.register_opt(rabbit_ssl_opt, group=rabbit_group) If no group is specified, options belong to the 'DEFAULT' section of config -files: +files:: glance-api.conf: [DEFAULT] @@ -169,13 +160,14 @@ files: password = guest virtual_host = / -Command-line options in a group are automatically prefixed with the group name: +Command-line options in a group are automatically prefixed with the +group name:: - --rabbit-host localhost --rabbit-use-ssl False + --rabbit-host localhost --rabbit-port 9999 Option values in the default group are referenced as attributes/properties on the config manager; groups are also attributes on the config manager, with -attributes for each of the options associated with the group: +attributes for each of the options associated with the group:: server.start(app, conf.bind_port, conf.bind_host, conf) @@ -184,7 +176,7 @@ attributes for each of the options associated with the group: port=conf.rabbit.port, ...) -Option values may reference other values using PEP 292 string substitution: +Option values may reference other values using PEP 292 string substitution:: opts = [ cfg.StrOpt('state_path', @@ -200,6 +192,22 @@ Option values may reference other values using PEP 292 string substitution: Note that interpolation can be avoided by using '$$'. +For command line utilities that dispatch to other command line utilities, the +disable_interspersed_args() method is available. If this this method is called, +then parsing e.g.:: + + script --verbose cmd --debug /tmp/mything + +will no longer return:: + + ['cmd', '/tmp/mything'] + +as the leftover arguments, but will instead return:: + + ['cmd', '--debug', '/tmp/mything'] + +i.e. argument parsing is stopped at the first non-option argument. + Options may be declared as secret so that their values are not leaked into log files: @@ -211,12 +219,15 @@ log files: """ -import sys -import ConfigParser +import collections import copy +import functools import optparse import os import string +import sys + +from glance.openstack.common import iniparser class Error(Exception): @@ -239,7 +250,7 @@ class ArgsAlreadyParsedError(Error): return ret -class NoSuchOptError(Error): +class NoSuchOptError(Error, AttributeError): """Raised if an opt which doesn't exist is referenced.""" def __init__(self, opt_name, group=None): @@ -288,8 +299,8 @@ class ConfigFilesNotFoundError(Error): self.config_files = config_files def __str__(self): - return 'Failed to read some config files: %s' % \ - string.join(self.config_files, ',') + return ('Failed to read some config files: %s' % + string.join(self.config_files, ',')) class ConfigFileParseError(Error): @@ -308,12 +319,16 @@ class ConfigFileValueError(Error): pass -def find_config_files(project=None, prog=None, filetype="conf"): +def find_config_files(project=None, prog=None, extension='.conf'): """Return a list of default configuration files. + :param project: an optional project name + :param prog: the program name, defaulting to the basename of sys.argv[0] + :param extension: the type of the config file + We default to two config files: [${project}.conf, ${prog}.conf] - And we look for those config files in the following directories: + And we look for those config files in the following directories:: ~/.${project}/ ~/ @@ -328,9 +343,6 @@ def find_config_files(project=None, prog=None, filetype="conf"): '~/.foo/bar.conf'] If no project name is supplied, we only look for ${prog.conf}. - - :param project: an optional project name - :param prog: the program name, defaulting to the basename of sys.argv[0] """ if prog is None: prog = os.path.basename(sys.argv[0]) @@ -342,23 +354,20 @@ def find_config_files(project=None, prog=None, filetype="conf"): fix_path('~'), os.path.join('/etc', project) if project else None, '/etc', - 'etc', + 'etc' ] cfg_dirs = filter(bool, cfg_dirs) - def search_dirs(dirs, basename): + def search_dirs(dirs, basename, extension): for d in dirs: - path = os.path.join(d, basename) + path = os.path.join(d, '%s%s' % (basename, extension)) if os.path.exists(path): return path config_files = [] - if project: - project_config = search_dirs(cfg_dirs, '%s.%s' % (project, filetype)) - config_files.append(project_config) - - config_files.append(search_dirs(cfg_dirs, '%s.%s' % (prog, filetype))) + config_files.append(search_dirs(cfg_dirs, project, extension)) + config_files.append(search_dirs(cfg_dirs, prog, extension)) return filter(bool, config_files) @@ -403,6 +412,7 @@ class Opt(object): help: an string explaining how the options value is used """ + multi = False def __init__(self, name, dest=None, short=None, default=None, metavar=None, help=None, secret=False): @@ -431,7 +441,7 @@ class Opt(object): self.secret = secret def _get_from_config_parser(self, cparser, section): - """Retrieves the option value from a ConfigParser object. + """Retrieves the option value from a MultiConfigParser object. This is the method ConfigOpts uses to look up the option value from config files. Most opt types override this method in order to perform @@ -542,9 +552,19 @@ class BoolOpt(Opt): 1/0, yes/no, true/false or on/off. """ + _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, + '0': False, 'no': False, 'false': False, 'off': False} + def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a boolean from ConfigParser.""" - return cparser.getboolean(section, self.dest) + def convert_bool(v): + value = self._boolean_states.get(v.lower()) + if value is None: + raise ValueError('Unexpected boolean value %r' % v) + + return value + + return [convert_bool(v) for v in cparser.get(section, self.dest)] def _add_to_cli(self, parser, group=None): """Extends the base class method to add the --nooptname option.""" @@ -571,7 +591,7 @@ class IntOpt(Opt): def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a integer from ConfigParser.""" - return cparser.getint(section, self.dest) + return [int(v) for v in cparser.get(section, self.dest)] def _get_optparse_kwargs(self, group, **kwargs): """Extends the base optparse keyword dict for integer options.""" @@ -585,7 +605,7 @@ class FloatOpt(Opt): def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a float from ConfigParser.""" - return cparser.getfloat(section, self.dest) + return [float(v) for v in cparser.get(section, self.dest)] def _get_optparse_kwargs(self, group, **kwargs): """Extends the base optparse keyword dict for float options.""" @@ -602,7 +622,7 @@ class ListOpt(Opt): def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a list from ConfigParser.""" - return cparser.get(section, self.dest).split(',') + return [v.split(',') for v in cparser.get(section, self.dest)] def _get_optparse_kwargs(self, group, **kwargs): """Extends the base optparse keyword dict for list options.""" @@ -624,14 +644,7 @@ class MultiStrOpt(Opt): Multistr opt values are string opts which may be specified multiple times. The opt value is a list containing all the string values specified. """ - - def _get_from_config_parser(self, cparser, section): - """Retrieve the opt value as a multistr from ConfigParser.""" - # FIXME(markmc): values spread across the CLI and multiple - # config files should be appended - value = \ - super(MultiStrOpt, self)._get_from_config_parser(cparser, section) - return value if value is None else [value] + multi = True def _get_optparse_kwargs(self, group, **kwargs): """Extends the base optparse keyword dict for multi str options.""" @@ -673,7 +686,7 @@ class OptGroup(object): self.title = title self.help = help - self._opts = {} # dict of dicts of {opt:, override:, default:) + self._opts = {} # dict of dicts of (opt:, override:, default:) self._optparse_group = None def _register_opt(self, opt): @@ -693,12 +706,75 @@ class OptGroup(object): def _get_optparse_group(self, parser): """Build an optparse.OptionGroup for this group.""" if self._optparse_group is None: - self._optparse_group = \ - optparse.OptionGroup(parser, self.title, self.help) + self._optparse_group = optparse.OptionGroup(parser, self.title, + self.help) return self._optparse_group -class ConfigOpts(object): +class ParseError(iniparser.ParseError): + def __init__(self, msg, lineno, line, filename): + super(ParseError, self).__init__(msg, lineno, line) + self.filename = filename + + def __str__(self): + return 'at %s:%d, %s: %r' % (self.filename, self.lineno, + self.msg, self.line) + + +class ConfigParser(iniparser.BaseParser): + def __init__(self, filename, sections): + super(ConfigParser, self).__init__() + self.filename = filename + self.sections = sections + self.section = None + + def parse(self): + with open(self.filename) as f: + return super(ConfigParser, self).parse(f) + + def new_section(self, section): + self.section = section + self.sections.setdefault(self.section, {}) + + def assignment(self, key, value): + if not self.section: + raise self.error_no_section() + + self.sections[self.section].setdefault(key, []) + self.sections[self.section][key].append('\n'.join(value)) + + def parse_exc(self, msg, lineno, line=None): + return ParseError(msg, lineno, line, self.filename) + + def error_no_section(self): + return self.parse_exc('Section must be started before assignment', + self.lineno) + + +class MultiConfigParser(object): + def __init__(self): + self.sections = {} + + def read(self, config_files): + read_ok = [] + + for filename in config_files: + parser = ConfigParser(filename, self.sections) + + try: + parser.parse() + except IOError: + continue + + read_ok.append(filename) + + return read_ok + + def get(self, section, name): + return self.sections[section][name] + + +class ConfigOpts(collections.Mapping): """ Config options which may be set on the command line or in config files. @@ -748,7 +824,9 @@ class ConfigOpts(object): usage=self.usage) self._cparser = None - self.register_cli_opt(\ + self.__cache = {} + + self.register_cli_opt( MultiStrOpt('config-file', default=self.default_config_files, metavar='PATH', @@ -757,6 +835,15 @@ class ConfigOpts(object): 'files taking precedence. The default files used ' 'are: %s' % (self.default_config_files, ))) + def __clear_cache(f): + @functools.wraps(f) + def __inner(self, *args, **kwargs): + if kwargs.pop('clear_cache', True): + self.__cache.clear() + return f(self, *args, **kwargs) + + return __inner + def __call__(self, args=None): """Parse command line arguments and config files. @@ -791,14 +878,33 @@ class ConfigOpts(object): :returns: the option value (after string subsititution) or a GroupAttr :raises: NoSuchOptError,ConfigFileValueError,TemplateSubstitutionError """ - return self._substitute(self._get(name)) + return self._get(name) + def __getitem__(self, key): + """Look up an option value and perform string substitution.""" + return self.__getattr__(key) + + def __contains__(self, key): + """Return True if key is the name of a registered opt or group.""" + return key in self._opts or key in self._groups + + def __iter__(self): + """Iterate over all registered opt and group names.""" + for key in self._opts.keys() + self._groups.keys(): + yield key + + def __len__(self): + """Return the number of options and option groups.""" + return len(self._opts) + len(self._groups) + + @__clear_cache def reset(self): """Reset the state of the object to before it was called.""" self._args = None self._cli_values = None self._cparser = None + @__clear_cache def register_opt(self, opt, group=None): """Register an option schema. @@ -821,11 +927,13 @@ class ConfigOpts(object): return True + @__clear_cache def register_opts(self, opts, group=None): """Register multiple option schemas at once.""" for opt in opts: - self.register_opt(opt, group) + self.register_opt(opt, group, clear_cache=False) + @__clear_cache def register_cli_opt(self, opt, group=None): """Register a CLI option schema. @@ -838,10 +946,10 @@ class ConfigOpts(object): :return: False if the opt was already register, True otherwise :raises: DuplicateOptError, ArgsAlreadyParsedError """ - if self._args != None: + if self._args is not None: raise ArgsAlreadyParsedError("cannot register CLI option") - if not self.register_opt(opt, group): + if not self.register_opt(opt, group, clear_cache=False): return False if group is not None: @@ -851,10 +959,11 @@ class ConfigOpts(object): return True + @__clear_cache def register_cli_opts(self, opts, group=None): """Register multiple CLI option schemas at once.""" for opt in opts: - self.register_cli_opt(opt, group) + self.register_cli_opt(opt, group, clear_cache=False) def register_group(self, group): """Register an option group. @@ -869,6 +978,7 @@ class ConfigOpts(object): self._groups[group.name] = copy.copy(group) + @__clear_cache def set_override(self, name, override, group=None): """Override an opt value. @@ -883,6 +993,7 @@ class ConfigOpts(object): opt_info = self._get_opt_info(name, group) opt_info['override'] = override + @__clear_cache def set_default(self, name, default, group=None): """Override an opt's default value. @@ -897,6 +1008,31 @@ class ConfigOpts(object): opt_info = self._get_opt_info(name, group) opt_info['default'] = default + def disable_interspersed_args(self): + """Set parsing to stop on the first non-option. + + If this this method is called, then parsing e.g. + + script --verbose cmd --debug /tmp/mything + + will no longer return: + + ['cmd', '/tmp/mything'] + + as the leftover arguments, but will instead return: + + ['cmd', '--debug', '/tmp/mything'] + + i.e. argument parsing is stopped at the first non-option argument. + """ + self._oparser.disable_interspersed_args() + + def enable_interspersed_args(self): + """Set parsing to not stop on the first non-option. + + This it the default behaviour.""" + self._oparser.enable_interspersed_args() + def log_opt_values(self, logger, lvl): """Log the value of all registered opts. @@ -923,7 +1059,7 @@ class ConfigOpts(object): _sanitize(opt, getattr(self, opt_name))) for group_name in self._groups: - group_attr = self.GroupAttr(self, group_name) + group_attr = self.GroupAttr(self, self._get_group(group_name)) for opt_name in sorted(self._groups[group_name]._opts): opt = self._get_opt_info(opt_name, group_name)['opt'] logger.log(lvl, "%-30s = %s", @@ -936,20 +1072,33 @@ class ConfigOpts(object): """Print the usage message for the current program.""" self._oparser.print_usage(file) + def print_help(self, file=None): + """Print the help message for the current program.""" + self._oparser.print_help(file) + def _get(self, name, group=None): + if isinstance(group, OptGroup): + key = (group.name, name) + else: + key = (group, name) + try: + return self.__cache[key] + except KeyError: + value = self._substitute(self._do_get(name, group)) + self.__cache[key] = value + return value + + def _do_get(self, name, group=None): """Look up an option value. :param name: the opt name (or 'dest', more precisely) - :param group: an option OptGroup + :param group: an OptGroup :returns: the option value, or a GroupAttr object :raises: NoSuchOptError, NoSuchGroupError, ConfigFileValueError, TemplateSubstitutionError """ if group is None and name in self._groups: - return self.GroupAttr(self, name) - - if group is not None: - group = self._get_group(group) + return self.GroupAttr(self, self._get_group(name)) info = self._get_opt_info(name, group) default, opt, override = map(lambda k: info[k], sorted(info.keys())) @@ -957,20 +1106,31 @@ class ConfigOpts(object): if override is not None: return override + values = [] if self._cparser is not None: section = group.name if group is not None else 'DEFAULT' try: - return opt._get_from_config_parser(self._cparser, section) - except (ConfigParser.NoOptionError, - ConfigParser.NoSectionError): + value = opt._get_from_config_parser(self._cparser, section) + except KeyError: pass - except ValueError, ve: + except ValueError as ve: raise ConfigFileValueError(str(ve)) + else: + if not opt.multi: + # No need to continue since the last value wins + return value[-1] + values.extend(value) name = name if group is None else group.name + '_' + name - value = self._cli_values.get(name, None) + value = self._cli_values.get(name) if value is not None: - return value + if not opt.multi: + return value + + return value + values + + if values: + return values if default is not None: return default @@ -1040,35 +1200,53 @@ class ConfigOpts(object): :raises: ConfigFilesNotFoundError, ConfigFileParseError """ - self._cparser = ConfigParser.SafeConfigParser() + self._cparser = MultiConfigParser() try: read_ok = self._cparser.read(config_files) - except ConfigParser.ParsingError, cpe: - raise ConfigFileParseError(cpe.filename, cpe.message) + except iniparser.ParseError as pe: + raise ConfigFileParseError(pe.filename, str(pe)) if read_ok != config_files: not_read_ok = filter(lambda f: f not in read_ok, config_files) raise ConfigFilesNotFoundError(not_read_ok) - class GroupAttr(object): + class GroupAttr(collections.Mapping): """ - A helper class representing the option values of a group as attributes. + A helper class representing the option values of a group as a mapping + and attributes. """ def __init__(self, conf, group): """Construct a GroupAttr object. :param conf: a ConfigOpts object - :param group: a group name or OptGroup object + :param group: an OptGroup object """ self.conf = conf self.group = group def __getattr__(self, name): """Look up an option value and perform template substitution.""" - return self.conf._substitute(self.conf._get(name, self.group)) + return self.conf._get(name, self.group) + + def __getitem__(self, key): + """Look up an option value and perform string substitution.""" + return self.__getattr__(key) + + def __contains__(self, key): + """Return True if key is the name of a registered opt or group.""" + return key in self.group._opts + + def __iter__(self): + """Iterate over all registered opt and group names.""" + for key in self.group._opts.keys(): + yield key + + def __len__(self): + """Return the number of options and option groups.""" + return len(self.group._opts) class StrSubWrapper(object): @@ -1099,8 +1277,7 @@ class ConfigOpts(object): class CommonConfigOpts(ConfigOpts): - DEFAULT_LOG_FORMAT = ('%(asctime)s %(process)d %(levelname)8s ' - '[%(name)s] %(message)s') + DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s" DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" common_cli_opts = [ diff --git a/glance/openstack/common/iniparser.py b/glance/openstack/common/iniparser.py new file mode 100644 index 0000000000..53ca023343 --- /dev/null +++ b/glance/openstack/common/iniparser.py @@ -0,0 +1,126 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class ParseError(Exception): + def __init__(self, message, lineno, line): + self.msg = message + self.line = line + self.lineno = lineno + + def __str__(self): + return 'at line %d, %s: %r' % (self.lineno, self.msg, self.line) + + +class BaseParser(object): + lineno = 0 + parse_exc = ParseError + + def _assignment(self, key, value): + self.assignment(key, value) + return None, [] + + def _get_section(self, line): + if line[-1] != ']': + return self.error_no_section_end_bracket(line) + if len(line) <= 2: + return self.error_no_section_name(line) + + return line[1:-1] + + def _split_key_value(self, line): + colon = line.find(':') + equal = line.find('=') + if colon < 0 and equal < 0: + return self.error_invalid_assignment(line) + + if colon < 0 or (equal >= 0 and equal < colon): + key, value = line[:equal], line[equal + 1:] + else: + key, value = line[:colon], line[colon + 1:] + + return key.strip(), [value.strip()] + + def parse(self, lineiter): + key = None + value = [] + + for line in lineiter: + self.lineno += 1 + + line = line.rstrip() + if not line: + # Blank line, ends multi-line values + if key: + key, value = self._assignment(key, value) + continue + elif line[0] in (' ', '\t'): + # Continuation of previous assignment + if key is None: + self.error_unexpected_continuation(line) + else: + value.append(line.lstrip()) + continue + + if key: + # Flush previous assignment, if any + key, value = self._assignment(key, value) + + if line[0] == '[': + # Section start + section = self._get_section(line) + if section: + self.new_section(section) + elif line[0] in '#;': + self.comment(line[1:].lstrip()) + else: + key, value = self._split_key_value(line) + if not key: + return self.error_empty_key(line) + + if key: + # Flush previous assignment, if any + self._assignment(key, value) + + def assignment(self, key, value): + """Called when a full assignment is parsed""" + raise NotImplementedError() + + def new_section(self, section): + """Called when a new section is started""" + raise NotImplementedError() + + def comment(self, comment): + """Called when a comment is parsed""" + pass + + def error_invalid_assignment(self, line): + raise self.parse_exc("No ':' or '=' found in assignment", + self.lineno, line) + + def error_empty_key(self, line): + raise self.parse_exc('Key cannot be empty', self.lineno, line) + + def error_unexpected_continuation(self, line): + raise self.parse_exc('Unexpected continuation line', + self.lineno, line) + + def error_no_section_end_bracket(self, line): + raise self.parse_exc('Invalid section (must end with ])', + self.lineno, line) + + def error_no_section_name(self, line): + raise self.parse_exc('Empty section name', self.lineno, line) diff --git a/glance/common/setup.py b/glance/openstack/common/setup.py similarity index 79% rename from glance/common/setup.py rename to glance/openstack/common/setup.py index 9eabfcca3f..60c731a9a2 100644 --- a/glance/common/setup.py +++ b/glance/openstack/common/setup.py @@ -37,8 +37,8 @@ def parse_mailmap(mailmap='.mailmap'): def canonicalize_emails(changelog, mapping): - """ Takes in a string and an email alias mapping and replaces all - instances of the aliases in the string with their real email + """Takes in a string and an email alias mapping and replaces all + instances of the aliases in the string with their real email. """ for alias, email in mapping.iteritems(): changelog = changelog.replace(alias, email) @@ -97,7 +97,7 @@ def _run_shell_command(cmd): def write_vcsversion(location): - """ Produce a vcsversion dict that mimics the old one produced by bzr + """Produce a vcsversion dict that mimics the old one produced by bzr. """ if os.path.isdir('.git'): branch_nick_cmd = 'git branch | grep -Ei "\* (.*)" | cut -f2 -d" "' @@ -118,10 +118,28 @@ version_info = { def write_git_changelog(): - """ Write a changelog based on the git changelog """ + """Write a changelog based on the git changelog.""" if os.path.isdir('.git'): git_log_cmd = 'git log --stat' changelog = _run_shell_command(git_log_cmd) mailmap = parse_mailmap() with open("ChangeLog", "w") as changelog_file: changelog_file.write(canonicalize_emails(changelog, mailmap)) + + +def generate_authors(): + """Create AUTHORS file using git commits.""" + jenkins_email = 'jenkins@review.openstack.org' + old_authors = 'AUTHORS.in' + new_authors = 'AUTHORS' + if os.path.isdir('.git'): + # don't include jenkins email address in AUTHORS file + git_log_cmd = "git log --format='%aN <%aE>' | sort -u | " \ + "grep -v " + jenkins_email + changelog = _run_shell_command(git_log_cmd) + mailmap = parse_mailmap() + with open(new_authors, 'w') as new_authors_fh: + new_authors_fh.write(canonicalize_emails(changelog, mailmap)) + if os.path.exists(old_authors): + with open(old_authors, "r") as old_authors_fh: + new_authors_fh.write('\n' + old_authors_fh.read()) diff --git a/glance/registry/__init__.py b/glance/registry/__init__.py index 685d52f283..7d18bf05f3 100644 --- a/glance/registry/__init__.py +++ b/glance/registry/__init__.py @@ -22,8 +22,8 @@ Registry API import logging import os -from glance.common import cfg from glance.common import exception +from glance.openstack.common import cfg from glance.registry import client logger = logging.getLogger('glance.registry') diff --git a/glance/registry/api/v1/images.py b/glance/registry/api/v1/images.py index 8ccb9461b8..a6c6cbe331 100644 --- a/glance/registry/api/v1/images.py +++ b/glance/registry/api/v1/images.py @@ -23,10 +23,10 @@ import logging from webob import exc -from glance.common import cfg from glance.common import exception from glance.common import utils from glance.common import wsgi +from glance.openstack.common import cfg from glance.registry.db import api as db_api diff --git a/glance/registry/db/__init__.py b/glance/registry/db/__init__.py index 36dbe30bda..abb304d8e8 100644 --- a/glance/registry/db/__init__.py +++ b/glance/registry/db/__init__.py @@ -17,7 +17,7 @@ # License for the specific language governing permissions and limitations # under the License. -from glance.common import cfg +from glance.openstack.common import cfg def add_options(conf): diff --git a/glance/registry/db/api.py b/glance/registry/db/api.py index 196179a0f7..6fb336d434 100644 --- a/glance/registry/db/api.py +++ b/glance/registry/db/api.py @@ -34,9 +34,9 @@ from sqlalchemy.orm import joinedload from sqlalchemy.orm import sessionmaker from sqlalchemy.sql import or_, and_ -from glance.common import cfg from glance.common import exception from glance.common import utils +from glance.openstack.common import cfg from glance.registry.db import migration from glance.registry.db import models diff --git a/glance/store/__init__.py b/glance/store/__init__.py index be5db8bc2b..a6729a8a3a 100644 --- a/glance/store/__init__.py +++ b/glance/store/__init__.py @@ -20,9 +20,9 @@ import os import sys import time -from glance.common import cfg from glance.common import exception from glance.common import utils +from glance.openstack.common import cfg from glance import registry from glance.store import location diff --git a/glance/store/filesystem.py b/glance/store/filesystem.py index 9e9324febf..ccdcf45ae0 100644 --- a/glance/store/filesystem.py +++ b/glance/store/filesystem.py @@ -25,9 +25,9 @@ import logging import os import urlparse -from glance.common import cfg from glance.common import exception from glance.common import utils +from glance.openstack.common import cfg import glance.store import glance.store.base import glance.store.location diff --git a/glance/store/rbd.py b/glance/store/rbd.py index 27dd22d6ab..21fcd8ccf1 100644 --- a/glance/store/rbd.py +++ b/glance/store/rbd.py @@ -24,8 +24,8 @@ import hashlib import logging import math -from glance.common import cfg from glance.common import exception +from glance.openstack.common import cfg import glance.store import glance.store.base import glance.store.location diff --git a/glance/store/s3.py b/glance/store/s3.py index 1df49f1a8c..73e820bf0d 100644 --- a/glance/store/s3.py +++ b/glance/store/s3.py @@ -24,9 +24,9 @@ import re import tempfile import urlparse -from glance.common import cfg from glance.common import exception from glance.common import utils +from glance.openstack.common import cfg import glance.store import glance.store.base import glance.store.location diff --git a/glance/store/scrubber.py b/glance/store/scrubber.py index e2a972642c..1118e7b95e 100644 --- a/glance/store/scrubber.py +++ b/glance/store/scrubber.py @@ -27,8 +27,8 @@ import glance.store.s3 import glance.store.swift from glance import registry from glance import store -from glance.common import cfg from glance.common import utils +from glance.openstack.common import cfg from glance.registry import client diff --git a/glance/store/swift.py b/glance/store/swift.py index 6db54c48eb..9e290910b4 100644 --- a/glance/store/swift.py +++ b/glance/store/swift.py @@ -25,8 +25,8 @@ import logging import math import urlparse -from glance.common import cfg from glance.common import exception +from glance.openstack.common import cfg import glance.store import glance.store.base import glance.store.location diff --git a/glance/tests/unit/test_cfg.py b/glance/tests/unit/test_cfg.py deleted file mode 100644 index 66de2f6305..0000000000 --- a/glance/tests/unit/test_cfg.py +++ /dev/null @@ -1,790 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import sys -import StringIO -import tempfile -import unittest - -import stubout - -from glance.common.cfg import * - - -class BaseTestCase(unittest.TestCase): - - def setUp(self): - self.conf = ConfigOpts(prog='test', - version='1.0', - usage='%prog FOO BAR', - default_config_files=[]) - self.tempfiles = [] - self.stubs = stubout.StubOutForTesting() - - def tearDown(self): - self.remove_tempfiles() - self.stubs.UnsetAll() - - def create_tempfiles(self, files): - for (basename, contents) in files: - (fd, path) = tempfile.mkstemp(prefix=basename) - self.tempfiles.append(path) - try: - os.write(fd, contents) - finally: - os.close(fd) - return self.tempfiles[-len(files):] - - def remove_tempfiles(self): - for p in self.tempfiles: - os.remove(p) - - -class LeftoversTestCase(BaseTestCase): - - def test_leftovers(self): - self.conf.register_cli_opt(StrOpt('foo')) - self.conf.register_cli_opt(StrOpt('bar')) - - leftovers = self.conf(['those', '--foo', 'this', - 'thems', '--bar', 'that', 'these']) - - self.assertEquals(leftovers, ['those', 'thems', 'these']) - - -class FindConfigFilesTestCase(BaseTestCase): - - def test_find_config_files(self): - config_files = \ - [os.path.expanduser('~/.blaa/blaa.conf'), '/etc/foo.conf'] - - self.stubs.Set(os.path, 'exists', lambda p: p in config_files) - - self.assertEquals(find_config_files(project='blaa', prog='foo'), - config_files) - - -class CliOptsTestCase(BaseTestCase): - - def _do_cli_test(self, opt_class, default, cli_args, value): - self.conf.register_cli_opt(opt_class('foo', default=default)) - - self.conf(cli_args) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, value) - - def test_str_default(self): - self._do_cli_test(StrOpt, None, [], None) - - def test_str_arg(self): - self._do_cli_test(StrOpt, None, ['--foo', 'bar'], 'bar') - - def test_bool_default(self): - self._do_cli_test(BoolOpt, False, [], False) - - def test_bool_arg(self): - self._do_cli_test(BoolOpt, None, ['--foo'], True) - - def test_bool_arg_inverse(self): - self._do_cli_test(BoolOpt, None, ['--foo', '--nofoo'], False) - - def test_int_default(self): - self._do_cli_test(IntOpt, 10, [], 10) - - def test_int_arg(self): - self._do_cli_test(IntOpt, None, ['--foo=20'], 20) - - def test_float_default(self): - self._do_cli_test(FloatOpt, 1.0, [], 1.0) - - def test_float_arg(self): - self._do_cli_test(FloatOpt, None, ['--foo', '2.0'], 2.0) - - def test_list_default(self): - self._do_cli_test(ListOpt, ['bar'], [], ['bar']) - - def test_list_arg(self): - self._do_cli_test(ListOpt, None, - ['--foo', 'blaa,bar'], ['blaa', 'bar']) - - def test_multistr_default(self): - self._do_cli_test(MultiStrOpt, ['bar'], [], ['bar']) - - def test_multistr_arg(self): - self._do_cli_test(MultiStrOpt, None, - ['--foo', 'blaa', '--foo', 'bar'], ['blaa', 'bar']) - - def test_help(self): - self.stubs.Set(sys, 'stdout', StringIO.StringIO()) - self.assertRaises(SystemExit, self.conf, ['--help']) - self.assertTrue('FOO BAR' in sys.stdout.getvalue()) - self.assertTrue('--version' in sys.stdout.getvalue()) - self.assertTrue('--help' in sys.stdout.getvalue()) - self.assertTrue('--config-file=PATH' in sys.stdout.getvalue()) - - def test_version(self): - self.stubs.Set(sys, 'stdout', StringIO.StringIO()) - self.assertRaises(SystemExit, self.conf, ['--version']) - self.assertTrue('1.0' in sys.stdout.getvalue()) - - def test_config_file(self): - paths = self.create_tempfiles([('1.conf', '[DEFAULT]'), - ('2.conf', '[DEFAULT]')]) - - self.conf(['--config-file', paths[0], '--config-file', paths[1]]) - - self.assertEquals(self.conf.config_file, paths) - - -class ConfigFileOptsTestCase(BaseTestCase): - - def test_str_default(self): - self.conf.register_opt(StrOpt('foo', default='bar')) - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n')]) - - self.conf(['--config-file', paths[0]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, 'bar') - - def test_str_value(self): - self.conf.register_opt(StrOpt('foo')) - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n' - 'foo = bar\n')]) - - self.conf(['--config-file', paths[0]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, 'bar') - - def test_str_value_override(self): - self.conf.register_cli_opt(StrOpt('foo')) - - paths = self.create_tempfiles([('1.conf', - '[DEFAULT]\n' - 'foo = baar\n'), - ('2.conf', - '[DEFAULT]\n' - 'foo = baaar\n')]) - - self.conf(['--foo', 'bar', - '--config-file', paths[0], - '--config-file', paths[1]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, 'baaar') - - def test_int_default(self): - self.conf.register_opt(IntOpt('foo', default=666)) - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n')]) - - self.conf(['--config-file', paths[0]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, 666) - - def test_int_value(self): - self.conf.register_opt(IntOpt('foo')) - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n' - 'foo = 666\n')]) - - self.conf(['--config-file', paths[0]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, 666) - - def test_int_value_override(self): - self.conf.register_cli_opt(IntOpt('foo')) - - paths = self.create_tempfiles([('1.conf', - '[DEFAULT]\n' - 'foo = 66\n'), - ('2.conf', - '[DEFAULT]\n' - 'foo = 666\n')]) - - self.conf(['--foo', '6', - '--config-file', paths[0], - '--config-file', paths[1]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, 666) - - def test_float_default(self): - self.conf.register_opt(FloatOpt('foo', default=6.66)) - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n')]) - - self.conf(['--config-file', paths[0]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, 6.66) - - def test_float_value(self): - self.conf.register_opt(FloatOpt('foo')) - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n' - 'foo = 6.66\n')]) - - self.conf(['--config-file', paths[0]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, 6.66) - - def test_float_value_override(self): - self.conf.register_cli_opt(FloatOpt('foo')) - - paths = self.create_tempfiles([('1.conf', - '[DEFAULT]\n' - 'foo = 6.6\n'), - ('2.conf', - '[DEFAULT]\n' - 'foo = 6.66\n')]) - - self.conf(['--foo', '6', - '--config-file', paths[0], - '--config-file', paths[1]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, 6.66) - - def test_list_default(self): - self.conf.register_opt(ListOpt('foo', default=['bar'])) - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n')]) - - self.conf(['--config-file', paths[0]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, ['bar']) - - def test_list_value(self): - self.conf.register_opt(ListOpt('foo')) - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n' - 'foo = bar\n')]) - - self.conf(['--config-file', paths[0]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, ['bar']) - - def test_list_value_override(self): - self.conf.register_cli_opt(ListOpt('foo')) - - paths = self.create_tempfiles([('1.conf', - '[DEFAULT]\n' - 'foo = bar,bar\n'), - ('2.conf', - '[DEFAULT]\n' - 'foo = b,a,r\n')]) - - self.conf(['--foo', 'bar', - '--config-file', paths[0], - '--config-file', paths[1]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, ['b', 'a', 'r']) - - def test_multistr_default(self): - self.conf.register_opt(MultiStrOpt('foo', default=['bar'])) - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n')]) - - self.conf(['--config-file', paths[0]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, ['bar']) - - def test_multistr_value(self): - self.conf.register_opt(MultiStrOpt('foo')) - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n' - 'foo = bar\n')]) - - self.conf(['--config-file', paths[0]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, ['bar']) - - def test_multistr_values_append(self): - self.conf.register_cli_opt(ListOpt('foo')) - - paths = self.create_tempfiles([('1.conf', - '[DEFAULT]\n' - 'foo = bar\n'), - ('2.conf', - '[DEFAULT]\n' - 'foo = bar\n')]) - - self.conf(['--foo', 'bar', - '--config-file', paths[0], - '--config-file', paths[1]]) - - self.assertTrue(hasattr(self.conf, 'foo')) - - # FIXME(markmc): values spread across the CLI and multiple - # config files should be appended - # self.assertEquals(self.conf.foo, ['bar', 'bar', 'bar']) - - -class OptGroupsTestCase(BaseTestCase): - - def test_arg_group(self): - blaa_group = OptGroup('blaa') - self.conf.register_group(blaa_group) - self.conf.register_cli_opt(StrOpt('foo'), group=blaa_group) - - self.conf(['--blaa-foo', 'bar']) - - self.assertTrue(hasattr(self.conf, 'blaa')) - self.assertTrue(hasattr(self.conf.blaa, 'foo')) - self.assertEquals(self.conf.blaa.foo, 'bar') - - def test_arg_group_by_name(self): - self.conf.register_group(OptGroup('blaa')) - self.conf.register_cli_opt(StrOpt('foo'), group='blaa') - - self.conf(['--blaa-foo', 'bar']) - - self.assertTrue(hasattr(self.conf, 'blaa')) - self.assertTrue(hasattr(self.conf.blaa, 'foo')) - self.assertEquals(self.conf.blaa.foo, 'bar') - - def test_arg_group_with_default(self): - self.conf.register_group(OptGroup('blaa')) - self.conf.register_cli_opt(StrOpt('foo', default='bar'), group='blaa') - - self.conf([]) - - self.assertTrue(hasattr(self.conf, 'blaa')) - self.assertTrue(hasattr(self.conf.blaa, 'foo')) - self.assertEquals(self.conf.blaa.foo, 'bar') - - def test_arg_group_in_config_file(self): - self.conf.register_group(OptGroup('blaa')) - self.conf.register_opt(StrOpt('foo'), group='blaa') - - paths = self.create_tempfiles([('test.conf', - '[blaa]\n' - 'foo = bar\n')]) - - self.conf(['--config-file', paths[0]]) - - self.assertTrue(hasattr(self.conf, 'blaa')) - self.assertTrue(hasattr(self.conf.blaa, 'foo')) - self.assertEquals(self.conf.blaa.foo, 'bar') - - -class TemplateSubstitutionTestCase(BaseTestCase): - - def _prep_test_str_sub(self, foo_default=None, bar_default=None): - self.conf.register_cli_opt(StrOpt('foo', default=foo_default)) - self.conf.register_cli_opt(StrOpt('bar', default=bar_default)) - - def _assert_str_sub(self): - self.assertTrue(hasattr(self.conf, 'bar')) - self.assertEquals(self.conf.bar, 'blaa') - - def test_str_sub_default_from_default(self): - self._prep_test_str_sub(foo_default='blaa', bar_default='$foo') - - self.conf([]) - - self._assert_str_sub() - - def test_str_sub_default_from_arg(self): - self._prep_test_str_sub(bar_default='$foo') - - self.conf(['--foo', 'blaa']) - - self._assert_str_sub() - - def test_str_sub_default_from_config_file(self): - self._prep_test_str_sub(bar_default='$foo') - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n' - 'foo = blaa\n')]) - - self.conf(['--config-file', paths[0]]) - - self._assert_str_sub() - - def test_str_sub_arg_from_default(self): - self._prep_test_str_sub(foo_default='blaa') - - self.conf(['--bar', '$foo']) - - self._assert_str_sub() - - def test_str_sub_arg_from_arg(self): - self._prep_test_str_sub() - - self.conf(['--foo', 'blaa', '--bar', '$foo']) - - self._assert_str_sub() - - def test_str_sub_arg_from_config_file(self): - self._prep_test_str_sub() - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n' - 'foo = blaa\n')]) - - self.conf(['--config-file', paths[0], '--bar=$foo']) - - self._assert_str_sub() - - def test_str_sub_config_file_from_default(self): - self._prep_test_str_sub(foo_default='blaa') - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n' - 'bar = $foo\n')]) - - self.conf(['--config-file', paths[0]]) - - self._assert_str_sub() - - def test_str_sub_config_file_from_arg(self): - self._prep_test_str_sub() - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n' - 'bar = $foo\n')]) - - self.conf(['--config-file', paths[0], '--foo=blaa']) - - self._assert_str_sub() - - def test_str_sub_config_file_from_config_file(self): - self._prep_test_str_sub() - - paths = self.create_tempfiles([('test.conf', - '[DEFAULT]\n' - 'bar = $foo\n' - 'foo = blaa\n')]) - - self.conf(['--config-file', paths[0]]) - - self._assert_str_sub() - - def test_str_sub_group_from_default(self): - self.conf.register_cli_opt(StrOpt('foo', default='blaa')) - self.conf.register_group(OptGroup('ba')) - self.conf.register_cli_opt(StrOpt('r', default='$foo'), group='ba') - - self.conf([]) - - self.assertTrue(hasattr(self.conf, 'ba')) - self.assertTrue(hasattr(self.conf.ba, 'r')) - self.assertEquals(self.conf.ba.r, 'blaa') - - -class ReparseTestCase(BaseTestCase): - - def test_reparse(self): - self.conf.register_group(OptGroup('blaa')) - self.conf.register_cli_opt(StrOpt('foo', default='r'), group='blaa') - - paths = self.create_tempfiles([('test.conf', - '[blaa]\n' - 'foo = b\n')]) - - self.conf(['--config-file', paths[0]]) - - self.assertTrue(hasattr(self.conf, 'blaa')) - self.assertTrue(hasattr(self.conf.blaa, 'foo')) - self.assertEquals(self.conf.blaa.foo, 'b') - - self.conf(['--blaa-foo', 'a']) - - self.assertTrue(hasattr(self.conf, 'blaa')) - self.assertTrue(hasattr(self.conf.blaa, 'foo')) - self.assertEquals(self.conf.blaa.foo, 'a') - - self.conf([]) - - self.assertTrue(hasattr(self.conf, 'blaa')) - self.assertTrue(hasattr(self.conf.blaa, 'foo')) - self.assertEquals(self.conf.blaa.foo, 'r') - - -class OverridesTestCase(BaseTestCase): - - def test_no_default_override(self): - self.conf.register_opt(StrOpt('foo')) - self.conf([]) - self.assertEquals(self.conf.foo, None) - self.conf.set_default('foo', 'bar') - self.assertEquals(self.conf.foo, 'bar') - - def test_default_override(self): - self.conf.register_opt(StrOpt('foo', default='foo')) - self.conf([]) - self.assertEquals(self.conf.foo, 'foo') - self.conf.set_default('foo', 'bar') - self.assertEquals(self.conf.foo, 'bar') - self.conf.set_default('foo', None) - self.assertEquals(self.conf.foo, 'foo') - - def test_override(self): - self.conf.register_opt(StrOpt('foo')) - self.conf.set_override('foo', 'bar') - self.conf([]) - self.assertEquals(self.conf.foo, 'bar') - - def test_group_no_default_override(self): - self.conf.register_group(OptGroup('blaa')) - self.conf.register_opt(StrOpt('foo'), group='blaa') - self.conf([]) - self.assertEquals(self.conf.blaa.foo, None) - self.conf.set_default('foo', 'bar', group='blaa') - self.assertEquals(self.conf.blaa.foo, 'bar') - - def test_default_override(self): - self.conf.register_group(OptGroup('blaa')) - self.conf.register_opt(StrOpt('foo', default='foo'), group='blaa') - self.conf([]) - self.assertEquals(self.conf.blaa.foo, 'foo') - self.conf.set_default('foo', 'bar', group='blaa') - self.assertEquals(self.conf.blaa.foo, 'bar') - self.conf.set_default('foo', None, group='blaa') - self.assertEquals(self.conf.blaa.foo, 'foo') - - def test_override(self): - self.conf.register_group(OptGroup('blaa')) - self.conf.register_opt(StrOpt('foo'), group='blaa') - self.conf.set_override('foo', 'bar', group='blaa') - self.conf([]) - self.assertEquals(self.conf.blaa.foo, 'bar') - - -class SadPathTestCase(BaseTestCase): - - def test_unknown_attr(self): - self.conf([]) - self.assertFalse(hasattr(self.conf, 'foo')) - self.assertRaises(NoSuchOptError, getattr, self.conf, 'foo') - - def test_unknown_group_attr(self): - self.conf.register_group(OptGroup('blaa')) - - self.conf([]) - - self.assertTrue(hasattr(self.conf, 'blaa')) - self.assertFalse(hasattr(self.conf.blaa, 'foo')) - self.assertRaises(NoSuchOptError, getattr, self.conf.blaa, 'foo') - - def test_ok_duplicate(self): - opt = StrOpt('foo') - self.conf.register_cli_opt(opt) - self.conf.register_cli_opt(opt) - - self.conf([]) - - self.assertTrue(hasattr(self.conf, 'foo')) - self.assertEquals(self.conf.foo, None) - - def test_error_duplicate(self): - self.conf.register_cli_opt(StrOpt('foo')) - self.assertRaises(DuplicateOptError, - self.conf.register_cli_opt, StrOpt('foo')) - - def test_error_duplicate_with_different_dest(self): - self.conf.register_cli_opt(StrOpt('foo', dest='f')) - self.assertRaises(DuplicateOptError, - self.conf.register_cli_opt, StrOpt('foo')) - - def test_error_duplicate_short(self): - self.conf.register_cli_opt(StrOpt('foo', short='f')) - self.assertRaises(DuplicateOptError, - self.conf.register_cli_opt, StrOpt('bar', short='f')) - - def test_no_such_group(self): - self.assertRaises(NoSuchGroupError, self.conf.register_cli_opt, - StrOpt('foo'), group='blaa') - - def test_already_parsed(self): - self.conf([]) - - self.assertRaises(ArgsAlreadyParsedError, - self.conf.register_cli_opt, StrOpt('foo')) - - def test_bad_cli_arg(self): - self.stubs.Set(sys, 'stderr', StringIO.StringIO()) - - self.assertRaises(SystemExit, self.conf, ['--foo']) - - self.assertTrue('error' in sys.stderr.getvalue()) - self.assertTrue('--foo' in sys.stderr.getvalue()) - - def _do_test_bad_cli_value(self, opt_class): - self.conf.register_cli_opt(opt_class('foo')) - - self.stubs.Set(sys, 'stderr', StringIO.StringIO()) - - self.assertRaises(SystemExit, self.conf, ['--foo', 'bar']) - - self.assertTrue('foo' in sys.stderr.getvalue()) - self.assertTrue('bar' in sys.stderr.getvalue()) - - def test_bad_int_arg(self): - self._do_test_bad_cli_value(IntOpt) - - def test_bad_float_arg(self): - self._do_test_bad_cli_value(FloatOpt) - - def test_conf_file_not_found(self): - paths = self.create_tempfiles([('test.conf', '')]) - os.remove(paths[0]) - self.tempfiles.remove(paths[0]) - - self.assertRaises(ConfigFilesNotFoundError, - self.conf, ['--config-file', paths[0]]) - - def test_conf_file_not_found(self): - paths = self.create_tempfiles([('test.conf', 'foo')]) - - self.assertRaises(ConfigFileParseError, - self.conf, ['--config-file', paths[0]]) - - def _do_test_conf_file_bad_value(self, opt_class): - self.conf.register_opt(opt_class('foo')) - - def test_conf_file_bad_bool(self): - self._do_test_conf_file_bad_value(BoolOpt) - - def test_conf_file_bad_int(self): - self._do_test_conf_file_bad_value(IntOpt) - - def test_conf_file_bad_float(self): - self._do_test_conf_file_bad_value(FloatOpt) - - def test_str_sub_from_group(self): - self.conf.register_group(OptGroup('f')) - self.conf.register_cli_opt(StrOpt('oo', default='blaa'), group='f') - self.conf.register_cli_opt(StrOpt('bar', default='$f.oo')) - - self.conf([]) - - self.assertFalse(hasattr(self.conf, 'bar')) - self.assertRaises(TemplateSubstitutionError, getattr, self.conf, 'bar') - - def test_set_default_unknown_attr(self): - self.conf([]) - self.assertRaises(NoSuchOptError, self.conf.set_default, 'foo', 'bar') - - def test_set_default_unknown_group(self): - self.conf([]) - self.assertRaises(NoSuchGroupError, - self.conf.set_default, 'foo', 'bar', group='blaa') - - def test_set_override_unknown_attr(self): - self.conf([]) - self.assertRaises(NoSuchOptError, self.conf.set_override, 'foo', 'bar') - - def test_set_override_unknown_group(self): - self.conf([]) - self.assertRaises(NoSuchGroupError, - self.conf.set_override, 'foo', 'bar', group='blaa') - - -class OptDumpingTestCase(BaseTestCase): - - class FakeLogger: - - def __init__(self, test_case, expected_lvl): - self.test_case = test_case - self.expected_lvl = expected_lvl - self.logged = [] - - def log(self, lvl, fmt, *args): - self.test_case.assertEquals(lvl, self.expected_lvl) - self.logged.append(fmt % args) - - def test_log_opt_values(self): - self.conf.register_cli_opt(StrOpt('foo')) - self.conf.register_cli_opt(StrOpt('passwd', secret=True)) - self.conf.register_group(OptGroup('blaa')) - self.conf.register_cli_opt(StrOpt('bar'), 'blaa') - self.conf.register_cli_opt(StrOpt('key', secret=True), 'blaa') - - self.conf(['--foo', 'this', '--blaa-bar', 'that', - '--blaa-key', 'admin', '--passwd', 'hush']) - - logger = self.FakeLogger(self, 666) - - self.conf.log_opt_values(logger, 666) - - self.assertEquals(logger.logged, [ - "*" * 80, - "Configuration options gathered from:", - "command line args: ['--foo', 'this', '--blaa-bar', 'that', "\ - "'--blaa-key', 'admin', '--passwd', 'hush']", - "config files: []", - "=" * 80, - "config_file = []", - "foo = this", - "passwd = ****", - "blaa.bar = that", - "blaa.key = *****", - "*" * 80, - ]) - - -class CommonOptsTestCase(BaseTestCase): - - def setUp(self): - super(CommonOptsTestCase, self).setUp() - self.conf = CommonConfigOpts() - - def test_debug_verbose(self): - self.conf(['--debug', '--verbose']) - - self.assertEquals(self.conf.debug, True) - self.assertEquals(self.conf.verbose, True) - - def test_logging_opts(self): - self.conf([]) - - self.assertTrue(self.conf.log_config is None) - self.assertTrue(self.conf.log_file is None) - self.assertTrue(self.conf.log_dir is None) - - self.assertEquals(self.conf.log_format, - CommonConfigOpts.DEFAULT_LOG_FORMAT) - self.assertEquals(self.conf.log_date_format, - CommonConfigOpts.DEFAULT_LOG_DATE_FORMAT) - - self.assertEquals(self.conf.use_syslog, False) diff --git a/glance/tests/unit/test_image_cache.py b/glance/tests/unit/test_image_cache.py index 707ef0c575..9661bb6423 100644 --- a/glance/tests/unit/test_image_cache.py +++ b/glance/tests/unit/test_image_cache.py @@ -25,8 +25,8 @@ import unittest import stubout from glance import image_cache -from glance.common import cfg from glance.common import utils +from glance.openstack.common import cfg from glance.tests import utils as test_utils from glance.tests.utils import skip_if_disabled, xattr_writes_supported diff --git a/glance/tests/unit/test_migrations.py b/glance/tests/unit/test_migrations.py index 15503c2b96..0c749cf8f3 100644 --- a/glance/tests/unit/test_migrations.py +++ b/glance/tests/unit/test_migrations.py @@ -34,8 +34,8 @@ from migrate.versioning.repository import Repository from sqlalchemy import * from sqlalchemy.pool import NullPool -from glance.common import cfg from glance.common import exception +from glance.openstack.common import cfg import glance.registry.db.migration as migration_api from glance.registry.db import models from glance.tests import utils diff --git a/openstack-common.conf b/openstack-common.conf new file mode 100644 index 0000000000..b018c97264 --- /dev/null +++ b/openstack-common.conf @@ -0,0 +1,7 @@ +[DEFAULT] + +# The list of modules to copy from openstack-common +modules=iniparser,cfg,setup + +# The base module to hold the copy of openstack.common +base=glance diff --git a/setup.py b/setup.py index 83877561de..bf5d61332e 100644 --- a/setup.py +++ b/setup.py @@ -21,8 +21,8 @@ import subprocess from setuptools import setup, find_packages from setuptools.command.sdist import sdist -from glance.common.setup import parse_requirements -from glance.common.setup import parse_dependency_links +from glance.openstack.common.setup import parse_requirements +from glance.openstack.common.setup import parse_dependency_links gettext.install('glance', unicode=1) diff --git a/tools/migrate_image_owners.py b/tools/migrate_image_owners.py index c5807891a7..ad76b7dddb 100644 --- a/tools/migrate_image_owners.py +++ b/tools/migrate_image_owners.py @@ -6,7 +6,8 @@ import sys import keystoneclient.v2_0.client import glance.common.context -import glance.common.cfg +import glance.openstack.common.cfg +import glance.registry.context import glance.registry.db.api as db_api @@ -56,18 +57,18 @@ def update_image_owners(image_owner_map, db, context): if __name__ == "__main__": - config = glance.common.cfg.CommonConfigOpts(project='glance', - prog='glance-registry') + config = glance.openstack.common.cfg.CommonConfigOpts(project='glance', + prog='glance-registry') extra_cli_opts = [ - glance.common.cfg.BoolOpt('dry-run', + glance.openstack.common.cfg.BoolOpt('dry-run', help='Print output but do not make db changes.'), - glance.common.cfg.StrOpt('keystone-auth-uri', + glance.openstack.common.cfg.StrOpt('keystone-auth-uri', help='Authentication endpoint'), - glance.common.cfg.StrOpt('keystone-admin-tenant-name', + glance.openstack.common.cfg.StrOpt('keystone-admin-tenant-name', help='Administrative user\'s tenant name'), - glance.common.cfg.StrOpt('keystone-admin-user', + glance.openstack.common.cfg.StrOpt('keystone-admin-user', help='Administrative user\'s id'), - glance.common.cfg.StrOpt('keystone-admin-password', + glance.openstack.common.cfg.StrOpt('keystone-admin-password', help='Administrative user\'s password'), ] config.register_cli_opts(extra_cli_opts)