From 57c4e9b6c686f1303c815698cc68b95bf9b033e7 Mon Sep 17 00:00:00 2001 From: Mark McLoughlin Date: Mon, 28 Nov 2011 14:37:58 +0000 Subject: [PATCH] Convert glance to use the new cfg module The changes here are substantial and widespread, but in summary: - We use cfg to parse the CLI and config files, rather than optparse and PasteDeploy - A schema is defined for all configuration options close to the code which uses the option - 2 ConfigOpts sub-classes are added to config.py basically just defining how to find config files; this means we can now use e.g. glance.conf for base config values which glance-api.conf can override - load_paste_app() is changed to load the paste app from the last config file in the stack and pass the app the ConfigOpts instance - The generic app and filter factories in wsgi.py are modified to pass a ConfigOpts instance to the apps and filters - A ConfigOpts subclass is added for the unit tests which writes out config values to a temporary config file and uses cfg to parse that I've tried to keep the switch as unobtrusive as possible leaving further cleanups for later e.g. - Moving PasteDeploy config out of the config files - I think it would be good to aim for having users modify the PasteDeploy config files only in fairly rare circumstances. To achieve this, we might define a number of common pipelines in the PasteDeploy config and allow the user to choose between those pipelines in the glance config. - We should add help strings to all the opts, even just for the sake of documenting them - We should move a bunch of the options into groups - e.g. all the rabbit options - We no longer rely on config files for default values, so the default config files could contain nothing but comments - i.e. explaining each option and showing what the default for it is - making it obvious where a user has explicitly set a value There are a couple of behavioural changes which I don't think are signifcant but are worth mentioning: - We used to support passing a config file as a positional argument but don't anymore; AFAICT, it was only used by glance-manage when launching servers and I've changed that to pass --config-file - log_opt_values() doesn't log unknown opts, so won't log any values for opts which get registered at runtime later Change-Id: Iafa998a2a8d860f1ad57e2cd2afee69686ed58ba --- bin/glance-api | 25 +- bin/glance-cache-cleaner | 25 +- bin/glance-cache-prefetcher | 24 +- bin/glance-cache-pruner | 25 +- bin/glance-cache-queue-image | 12 +- bin/glance-control | 63 ++-- bin/glance-manage | 64 ++-- bin/glance-registry | 25 +- bin/glance-scrubber | 49 +-- glance/api/middleware/cache.py | 2 +- glance/api/middleware/cache_manage.py | 3 +- glance/api/middleware/version_negotiation.py | 2 +- glance/api/v1/images.py | 6 +- glance/api/v1/router.py | 2 +- glance/api/versions.py | 5 +- glance/common/config.py | 291 +++--------------- glance/common/context.py | 30 +- glance/common/notifier.py | 48 +-- glance/common/wsgi.py | 172 +++++++++-- glance/image_cache/__init__.py | 19 +- glance/image_cache/cleaner.py | 2 +- glance/image_cache/drivers/base.py | 10 +- glance/image_cache/drivers/sqlite.py | 20 +- glance/image_cache/drivers/xattr.py | 11 +- glance/image_cache/prefetcher.py | 9 +- glance/image_cache/pruner.py | 2 +- glance/image_cache/queue_image.py | 9 +- glance/registry/__init__.py | 51 ++- glance/registry/api/v1/__init__.py | 2 +- glance/registry/api/v1/images.py | 29 +- glance/registry/db/__init__.py | 27 +- glance/registry/db/api.py | 21 +- glance/registry/db/migration.py | 10 +- glance/store/__init__.py | 28 +- glance/store/base.py | 4 +- glance/store/filesystem.py | 22 +- glance/store/rbd.py | 26 +- glance/store/s3.py | 26 +- glance/store/scrubber.py | 32 +- glance/store/swift.py | 61 ++-- .../test_bin_glance_cache_manage.py | 3 +- .../tests/functional/test_cache_middleware.py | 3 +- .../tests/functional/test_client_redirects.py | 5 +- glance/tests/stubs.py | 27 +- glance/tests/unit/test_api.py | 24 +- glance/tests/unit/test_clients.py | 7 +- glance/tests/unit/test_config.py | 134 +------- glance/tests/unit/test_filesystem_store.py | 3 +- glance/tests/unit/test_http_store.py | 3 +- glance/tests/unit/test_image_cache.py | 26 +- glance/tests/unit/test_migrations.py | 13 +- glance/tests/unit/test_misc.py | 4 +- glance/tests/unit/test_notifier.py | 9 +- glance/tests/unit/test_s3_store.py | 7 +- glance/tests/unit/test_store_location.py | 3 +- glance/tests/unit/test_swift_store.py | 15 +- glance/tests/unit/test_versions.py | 8 +- glance/tests/utils.py | 35 +++ 58 files changed, 716 insertions(+), 907 deletions(-) diff --git a/bin/glance-api b/bin/glance-api index 2bc85086de..d2748a20ea 100755 --- a/bin/glance-api +++ b/bin/glance-api @@ -23,7 +23,6 @@ Glance API Server """ import gettext -import optparse import os import sys @@ -37,33 +36,19 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): gettext.install('glance', unicode=1) -from glance import version from glance.common import config from glance.common import wsgi -def create_options(parser): - """ - Sets up the CLI and config-file options that may be - parsed and program commands. - - :param parser: The option parser - """ - config.add_common_options(parser) - config.add_log_options(parser) - - if __name__ == '__main__': - oparser = optparse.OptionParser(version='%%prog %s' - % version.version_string()) - create_options(oparser) - (options, args) = config.parse_options(oparser) - try: - conf, app = config.load_paste_app('glance-api', options, args) + conf = config.GlanceConfigOpts() + conf() + + app = config.load_paste_app(conf) server = wsgi.Server() - server.start(app, int(conf['bind_port']), conf['bind_host'], conf) + server.start(app, conf, default_port=9292) server.wait() except RuntimeError, e: sys.exit("ERROR: %s" % e) diff --git a/bin/glance-cache-cleaner b/bin/glance-cache-cleaner index 065b5fe3e0..d5a8bf3677 100755 --- a/bin/glance-cache-cleaner +++ b/bin/glance-cache-cleaner @@ -33,7 +33,6 @@ period, we automatically sweep it up. """ import gettext -import optparse import os import sys @@ -47,31 +46,15 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): gettext.install('glance', unicode=1) -from glance import version from glance.common import config -from glance.common import wsgi - - -def create_options(parser): - """ - Sets up the CLI and config-file options that may be - parsed and program commands. - - :param parser: The option parser - """ - config.add_common_options(parser) - config.add_log_options(parser) if __name__ == '__main__': - oparser = optparse.OptionParser(version='%%prog %s' - % version.version_string()) - create_options(oparser) - (options, args) = config.parse_options(oparser) - try: - conf, app = config.load_paste_app('glance-cleaner', options, args, - 'glance-cache') + conf = config.GlanceCacheConfigOpts() + conf() + + app = config.load_paste_app(conf, 'glance-cleaner') app.run() except RuntimeError, e: sys.exit("ERROR: %s" % e) diff --git a/bin/glance-cache-prefetcher b/bin/glance-cache-prefetcher index 8ab4475b87..bd2b8cb697 100755 --- a/bin/glance-cache-prefetcher +++ b/bin/glance-cache-prefetcher @@ -24,7 +24,6 @@ images to be pretched. """ import gettext -import optparse import os import sys @@ -38,30 +37,15 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): gettext.install('glance', unicode=1) -from glance import version from glance.common import config -def create_options(parser): - """ - Sets up the CLI and config-file options that may be - parsed and program commands. - - :param parser: The option parser - """ - config.add_common_options(parser) - config.add_log_options(parser) - - if __name__ == '__main__': - oparser = optparse.OptionParser(version='%%prog %s' - % version.version_string()) - create_options(oparser) - (options, args) = config.parse_options(oparser) - try: - conf, app = config.load_paste_app('glance-prefetcher', options, args, - 'glance-cache') + conf = config.GlanceCacheConfigOpts() + conf() + + app = config.load_paste_app(conf, 'glance-prefetcher') app.run() except RuntimeError, e: sys.exit("ERROR: %s" % e) diff --git a/bin/glance-cache-pruner b/bin/glance-cache-pruner index 56bcf0e375..9d6a3d00ed 100755 --- a/bin/glance-cache-pruner +++ b/bin/glance-cache-pruner @@ -25,7 +25,6 @@ This is meant to be run as a periodic task, perhaps every half-hour. """ import gettext -import optparse import os import sys @@ -39,31 +38,15 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): gettext.install('glance', unicode=1) -from glance import version from glance.common import config -from glance.common import wsgi - - -def create_options(parser): - """ - Sets up the CLI and config-file options that may be - parsed and program commands. - - :param parser: The option parser - """ - config.add_common_options(parser) - config.add_log_options(parser) if __name__ == '__main__': - oparser = optparse.OptionParser(version='%%prog %s' - % version.version_string()) - create_options(oparser) - (options, args) = config.parse_options(oparser) - try: - conf, app = config.load_paste_app('glance-pruner', options, args, - 'glance-cache') + conf = config.GlanceCacheConfigOpts() + conf() + + app = config.load_paste_app(conf, 'glance-pruner') app.run() except RuntimeError, e: sys.exit("ERROR: %s" % e) diff --git a/bin/glance-cache-queue-image b/bin/glance-cache-queue-image index 0975c37d3f..2bfd68709e 100644 --- a/bin/glance-cache-queue-image +++ b/bin/glance-cache-queue-image @@ -21,7 +21,6 @@ CLI Utility to queue one or more images for prefetching into the image cache """ import gettext -import optparse import os import sys @@ -35,7 +34,6 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): gettext.install('glance', unicode=1) -from glance import version from glance.common import config USAGE = """ @@ -44,13 +42,11 @@ USAGE = """ if __name__ == '__main__': - oparser = optparse.OptionParser(version='%%prog %s' - % version.version_string(), - usage=USAGE) - (options, args) = config.parse_options(oparser) - try: - conf, app = config.load_paste_app('glance-queue-image', options, args) + conf = config.GlanceCacheConfigOpts(usage=USAGE) + args = conf() + + app = config.load_paste_app(conf, 'glance-queue-image') app.run(args) except RuntimeError, e: sys.exit("ERROR: %s" % e) diff --git a/bin/glance-control b/bin/glance-control index 8c25d350eb..39d588d915 100755 --- a/bin/glance-control +++ b/bin/glance-control @@ -43,6 +43,7 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): gettext.install('glance', unicode=1) from glance import version +from glance.common import cfg from glance.common import config ALL_COMMANDS = ['start', 'stop', 'shutdown', 'restart', @@ -65,11 +66,11 @@ And command is one of: And CONFPATH is the optional configuration file to use.""" -def pid_files(server, options): +def pid_files(server, conf): pid_files = [] - if options['pid_file']: - if os.path.exists(os.path.abspath(options['pid_file'])): - pid_files = [os.path.abspath(options['pid_file'])] + if conf.pid_file: + if os.path.exists(os.path.abspath(conf.pid_file)): + pid_files = [os.path.abspath(conf.pid_file)] else: if os.path.exists('/var/run/glance/%s.pid' % server): pid_files = ['/var/run/glance/%s.pid' % server] @@ -78,10 +79,10 @@ def pid_files(server, options): yield pid_file, pid -def do_start(server, options, args): +def do_start(server, conf, args): server_type = '-'.join(server.split('-')[:-1]) - for pid_file, pid in pid_files(server, options): + for pid_file, pid in pid_files(server, conf): if os.path.exists('/proc/%s' % pid): print "%s appears to already be running: %s" % (server, pid_file) return @@ -112,7 +113,6 @@ def do_start(server, options, args): fp.close() def launch(ini_file, pid_file): - args = [server, ini_file] print 'Starting %s with %s' % (server, ini_file) pid = os.fork() @@ -125,7 +125,7 @@ def do_start(server, options, args): except OSError: pass try: - os.execlp('%s' % server, server, ini_file) + os.execlp('%s' % server, server, '--config-file', ini_file) except OSError, e: sys.exit('unable to launch %s. Got error: %s' % (server, "%s" % e)) @@ -133,31 +133,31 @@ def do_start(server, options, args): else: write_pid_file(pid_file, pid) - if not options['pid_file']: + if not conf.pid_file: pid_file = '/var/run/glance/%s.pid' % server else: - pid_file = os.path.abspath(options['pid_file']) + pid_file = os.path.abspath(conf.pid_file) try: - conf_file = config.find_config_file(server, options, args) + if args and os.path.exists(args[0]): + conf_file = os.path.abspath(os.path.expanduser(args[0])) + else: + # Assume paste config is in the last config file + conf_file = conf.config_file[-1] except RuntimeError, err: sys.exit("Could not find any configuration file to use: %s" % err) - launch_args = [(conf_file, pid_file)] - - # start all servers - for conf_file, pid_file in launch_args: - launch(conf_file, pid_file) + launch(conf_file, pid_file) -def do_stop(server, options, args, graceful=False): +def do_stop(server, conf, args, graceful=False): if graceful and server in GRACEFUL_SHUTDOWN_SERVERS: sig = signal.SIGHUP else: sig = signal.SIGTERM did_anything = False - pfiles = pid_files(server, options) + pfiles = pid_files(server, conf) for pid_file, pid in pfiles: did_anything = True try: @@ -182,13 +182,12 @@ def do_stop(server, options, args, graceful=False): if __name__ == '__main__': - oparser = optparse.OptionParser(usage=USAGE, version='%%prog %s' - % version.version_string()) - oparser.add_option('--pid-file', default=None, metavar="PATH", - help="File to use as pid file. Default: " - "/var/run/glance/$server.pid") - config.add_common_options(oparser) - (options, args) = config.parse_options(oparser) + conf = config.GlanceConfigOpts(usage=USAGE) + conf.register_cli_opt(cfg.StrOpt('pid-file', + metavar='PATH', + help='File to use as pid file. Default: ' + '/var/run/glance/$server.pid')) + args = conf() if len(args) < 2: oparser.print_usage() @@ -217,23 +216,23 @@ if __name__ == '__main__': if command == 'start': for server in servers: - do_start(server, options, args) + do_start(server, conf, args) if command == 'stop': for server in servers: - do_stop(server, options, args) + do_stop(server, conf, args) if command == 'shutdown': for server in servers: - do_stop(server, options, args, graceful=True) + do_stop(server, conf, args, graceful=True) if command == 'restart': for server in servers: - do_stop(server, options, args) + do_stop(server, conf, args) for server in servers: - do_start(server, options, args) + do_start(server, conf, args) if command == 'reload' or command == 'force-reload': for server in servers: - do_stop(server, options, args, graceful=True) - do_start(server, options, args) + do_stop(server, conf, args, graceful=True) + do_start(server, conf, args) diff --git a/bin/glance-manage b/bin/glance-manage index afd9e74ca3..b0ccfb2fee 100755 --- a/bin/glance-manage +++ b/bin/glance-manage @@ -27,7 +27,6 @@ Glance Management Utility # glance-manage (or the other way around) import gettext -import optparse import os import sys @@ -41,41 +40,29 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): gettext.install('glance', unicode=1) -from glance import version as glance_version +from glance.common import cfg from glance.common import config from glance.common import exception import glance.registry.db import glance.registry.db.migration -def create_options(parser): - """ - Sets up the CLI and config-file options that may be - parsed and program commands. - - :param parser: The option parser - """ - config.add_common_options(parser) - config.add_log_options(parser) - glance.registry.db.add_options(parser) - - -def do_db_version(options, args): +def do_db_version(conf, args): """Print database's current migration level""" - print glance.registry.db.migration.db_version(options) + print glance.registry.db.migration.db_version(conf) -def do_upgrade(options, args): +def do_upgrade(conf, args): """Upgrade the database's migration level""" try: db_version = args[1] except IndexError: db_version = None - glance.registry.db.migration.upgrade(options, version=db_version) + glance.registry.db.migration.upgrade(conf, version=db_version) -def do_downgrade(options, args): +def do_downgrade(conf, args): """Downgrade the database's migration level""" try: db_version = args[1] @@ -83,24 +70,24 @@ def do_downgrade(options, args): raise exception.MissingArgumentError( "downgrade requires a version argument") - glance.registry.db.migration.downgrade(options, version=db_version) + glance.registry.db.migration.downgrade(conf, version=db_version) -def do_version_control(options, args): +def do_version_control(conf, args): """Place a database under migration control""" - glance.registry.db.migration.version_control(options) + glance.registry.db.migration.version_control(conf) -def do_db_sync(options, args): +def do_db_sync(conf, args): """Place a database under migration control and upgrade""" try: db_version = args[1] except IndexError: db_version = None - glance.registry.db.migration.db_sync(options, version=db_version) + glance.registry.db.migration.db_sync(conf, version=db_version) -def dispatch_cmd(options, args): +def dispatch_cmd(conf, args): """Search for do_* cmd in this module and then run it""" cmd = args[0] try: @@ -109,35 +96,32 @@ def dispatch_cmd(options, args): sys.exit("ERROR: unrecognized command '%s'" % cmd) try: - cmd_func(options, args) + cmd_func(conf, args) except exception.GlanceException, e: sys.exit("ERROR: %s" % e) def main(): - version = '%%prog %s' % glance_version.version_string() - usage = "%prog [options] " - oparser = optparse.OptionParser(usage, version=version) - create_options(oparser) - (options, args) = config.parse_options(oparser) - try: # We load the glance-registry config section because # sql_connection is only part of the glance registry. - conf_file = config.find_config_file('glance-registry', options, args) - conf = config.load_paste_config(conf_file, 'glance-registry') - config.setup_logging(options, conf) + default_config_files = \ + cfg.find_config_files(project='glance', prog='glance-registry') + + conf = \ + config.GlanceConfigOpts(default_config_files=default_config_files, + usage="%prog [options] ") + glance.registry.db.add_options(conf) + args = conf() + config.setup_logging(conf) except RuntimeError, e: sys.exit("ERROR: %s" % e) if not args: - oparser.print_usage() + conf.print_usage() sys.exit(1) - if conf.get('sql_connection') and not options['sql_connection']: - options['sql_connection'] = conf.get('sql_connection') - - dispatch_cmd(options, args) + dispatch_cmd(conf, args) if __name__ == '__main__': diff --git a/bin/glance-registry b/bin/glance-registry index 0683267677..6f10226948 100755 --- a/bin/glance-registry +++ b/bin/glance-registry @@ -23,7 +23,6 @@ Reference implementation server for Glance Registry """ import gettext -import optparse import os import sys @@ -37,33 +36,19 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): gettext.install('glance', unicode=1) -from glance import version from glance.common import config from glance.common import wsgi -def create_options(parser): - """ - Sets up the CLI and config-file options that may be - parsed and program commands. - - :param parser: The option parser - """ - config.add_common_options(parser) - config.add_log_options(parser) - - if __name__ == '__main__': - oparser = optparse.OptionParser(version='%%prog %s' - % version.version_string()) - create_options(oparser) - (options, args) = config.parse_options(oparser) - try: - conf, app = config.load_paste_app('glance-registry', options, args) + conf = config.GlanceConfigOpts() + conf() + + app = config.load_paste_app(conf) server = wsgi.Server() - server.start(app, int(conf['bind_port']), conf['bind_host'], conf) + server.start(app, conf, default_port=9191) server.wait() except RuntimeError, e: sys.exit("ERROR: %s" % e) diff --git a/bin/glance-scrubber b/bin/glance-scrubber index 4eb5fce0f7..1e9ffe420e 100755 --- a/bin/glance-scrubber +++ b/bin/glance-scrubber @@ -21,7 +21,6 @@ Glance Scrub Service """ import gettext -import optparse import os import sys @@ -35,44 +34,30 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): gettext.install('glance', unicode=1) -from glance import version +from glance.common import cfg from glance.common import config from glance.store import scrubber -def create_options(parser): - """ - Sets up the CLI and config-file options that may be - parsed and program commands. - - :param parser: The option parser - """ - config.add_common_options(parser) - config.add_log_options(parser) - parser.add_option("-D", "--daemon", default=False, dest="daemon", - action="store_true", - help="Run as a long-running process. When not " - "specified (the default) run the scrub " - "operation once and then exits. When specified " - "do not exit and run scrub on wakeup_time " - "interval as specified in the config file.") - - if __name__ == '__main__': - oparser = optparse.OptionParser(version='%%prog %s' - % version.version_string()) - create_options(oparser) - (options, args) = config.parse_options(oparser) - try: - conf, app = config.load_paste_app('glance-scrubber', options, args) - daemon = options.get('daemon') or \ - config.get_option(conf, 'daemon', type='bool', - default=False) + conf = config.GlanceConfigOpts() + conf.register_cli_opt( + cfg.BoolOpt('daemon', + short='D', + default=False, + help='Run as a long-running process. When not ' + 'specified (the default) run the scrub operation ' + 'once and then exits. When specified do not exit ' + 'and run scrub on wakeup_time interval as ' + 'specified in the config.')) + conf.register_opt(cfg.IntOpt('wakeup_time', default=300)) + conf() - if daemon: - wakeup_time = int(conf.get('wakeup_time', 300)) - server = scrubber.Daemon(wakeup_time) + app = config.load_paste_app(conf, 'glance-scrubber') + + if conf.daemon: + server = scrubber.Daemon(conf.wakeup_time) server.start(app) server.wait() else: diff --git a/glance/api/middleware/cache.py b/glance/api/middleware/cache.py index f236abc23b..f4dfda9552 100644 --- a/glance/api/middleware/cache.py +++ b/glance/api/middleware/cache.py @@ -43,7 +43,7 @@ get_images_re = re.compile(r'^(/v\d+)*/images/(.+)$') class CacheFilter(wsgi.Middleware): - def __init__(self, app, conf): + def __init__(self, app, conf, **local_conf): self.conf = conf self.cache = image_cache.ImageCache(conf) self.serializer = images.ImageSerializer() diff --git a/glance/api/middleware/cache_manage.py b/glance/api/middleware/cache_manage.py index 07ae2cb94c..cb8e4c2323 100644 --- a/glance/api/middleware/cache_manage.py +++ b/glance/api/middleware/cache_manage.py @@ -28,8 +28,7 @@ logger = logging.getLogger(__name__) class CacheManageFilter(wsgi.Middleware): - def __init__(self, app, conf): - + def __init__(self, app, conf, **local_conf): map = app.map resource = cached_images.create_resource(conf) diff --git a/glance/api/middleware/version_negotiation.py b/glance/api/middleware/version_negotiation.py index 73d597ed59..95e005ca41 100644 --- a/glance/api/middleware/version_negotiation.py +++ b/glance/api/middleware/version_negotiation.py @@ -35,7 +35,7 @@ logger = logging.getLogger('glance.api.middleware.version_negotiation') class VersionNegotiationFilter(wsgi.Middleware): - def __init__(self, app, conf): + def __init__(self, app, conf, **local_conf): self.versions_app = versions.Controller(conf) self.version_uri_regex = re.compile(r"^v(\d+)\.?(\d+)?") self.conf = conf diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index 71f7a999fe..fc904d94cd 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -34,6 +34,7 @@ from webob.exc import (HTTPNotFound, import glance.api.v1 from glance.api.v1 import controller from glance import image_cache +from glance.common import cfg from glance.common import exception from glance.common import notifier from glance.common import wsgi @@ -76,8 +77,11 @@ class Controller(controller.BaseController): DELETE /images/ -- Delete the image with id """ + default_store_opt = cfg.StrOpt('default_store', default='file') + def __init__(self, conf): self.conf = conf + self.conf.register_opt(self.default_store_opt) glance.store.create_stores(conf) self.notifier = notifier.Notifier(conf) registry.configure_registry_client(conf) @@ -290,7 +294,7 @@ class Controller(controller.BaseController): raise HTTPBadRequest(explanation=msg) store_name = req.headers.get('x-image-meta-store', - self.conf['default_store']) + self.conf.default_store) store = self.get_store_or_400(req, store_name) diff --git a/glance/api/v1/router.py b/glance/api/v1/router.py index 6e0ac3dc1e..3d0a02c8a2 100644 --- a/glance/api/v1/router.py +++ b/glance/api/v1/router.py @@ -30,7 +30,7 @@ class API(wsgi.Router): """WSGI router for Glance v1 API requests.""" - def __init__(self, conf): + def __init__(self, conf, **local_conf): self.conf = conf mapper = routes.Mapper() diff --git a/glance/api/versions.py b/glance/api/versions.py index 924e41525c..5dfbc5caf9 100644 --- a/glance/api/versions.py +++ b/glance/api/versions.py @@ -24,6 +24,8 @@ import json import webob.dec +from glance.common import wsgi + class Controller(object): @@ -63,5 +65,4 @@ class Controller(object): return response def get_href(self): - return "http://%s:%s/v1/" % (self.conf['bind_host'], - self.conf['bind_port']) + return "http://%s:%s/v1/" % wsgi.get_bind_addr(self.conf, 9292) diff --git a/glance/common/config.py b/glance/common/config.py index 327e76b894..84b988ce42 100644 --- a/glance/common/config.py +++ b/glance/common/config.py @@ -20,159 +20,68 @@ Routines for configuring Glance """ -import ConfigParser import logging import logging.config import logging.handlers -import optparse import os -import re import sys -from paste import deploy - -import glance.common.exception as exception - -DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s" -DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" +from glance import version +from glance.common import cfg +from glance.common import utils +from glance.common import wsgi -def parse_options(parser, cli_args=None): - """ - Returns the parsed CLI options, command to run and its arguments, merged - with any same-named options found in a configuration file. +class GlanceConfigOpts(cfg.CommonConfigOpts): - The function returns a tuple of (options, args), where options is a - mapping of option key/str(value) pairs, and args is the set of arguments - (not options) supplied on the command-line. - - The reason that the option values are returned as strings only is that - ConfigParser and paste.deploy only accept string values... - - :param parser: The option parser - :param cli_args: (Optional) Set of arguments to process. If not present, - sys.argv[1:] is used. - :retval tuple of (options, args) - """ - - (options, args) = parser.parse_args(cli_args) - - return (vars(options), args) + def __init__(self, default_config_files=None, **kwargs): + super(GlanceConfigOpts, self).__init__( + project='glance', + version='%%prog %s' % version.version_string(), + default_config_files=default_config_files, + **kwargs) -def add_common_options(parser): - """ - Given a supplied optparse.OptionParser, adds an OptionGroup that - represents all common configuration options. +class GlanceCacheConfigOpts(GlanceConfigOpts): - :param parser: optparse.OptionParser - """ - help_text = "The following configuration options are common to "\ - "all glance programs." - - group = optparse.OptionGroup(parser, "Common Options", help_text) - group.add_option('-v', '--verbose', default=False, dest="verbose", - action="store_true", - help="Print more verbose output") - group.add_option('-d', '--debug', default=False, dest="debug", - action="store_true", - help="Print debugging output") - group.add_option('--config-file', default=None, metavar="PATH", - help="Path to the config file to use. When not specified " - "(the default), we generally look at the first " - "argument specified to be a config file, and if " - "that is also missing, we search standard " - "directories for a config file.") - parser.add_option_group(group) + def __init__(self, **kwargs): + config_files = cfg.find_config_files(project='glance', + prog='glance-cache') + super(GlanceCacheConfigOpts, self).__init__(config_files, **kwargs) -def add_log_options(parser): - """ - Given a supplied optparse.OptionParser, adds an OptionGroup that - represents all the configuration options around logging. - - :param parser: optparse.OptionParser - """ - help_text = "The following configuration options are specific to logging "\ - "functionality for this program." - - group = optparse.OptionGroup(parser, "Logging Options", help_text) - group.add_option('--log-config', default=None, metavar="PATH", - help="If this option is specified, the logging " - "configuration file specified is used and overrides " - "any other logging options specified. Please see " - "the Python logging module documentation for " - "details on logging configuration files.") - group.add_option('--log-date-format', metavar="FORMAT", - default=DEFAULT_LOG_DATE_FORMAT, - help="Format string for %(asctime)s in log records. " - "Default: %default") - group.add_option('--log-file', default=None, metavar="PATH", - help="(Optional) Name of log file to output to. " - "If not set, logging will go to stdout.") - group.add_option("--log-dir", default=None, - help="(Optional) The directory to keep log files in " - "(will be prepended to --logfile)") - group.add_option('--use-syslog', default=False, dest="use_syslog", - action="store_true", - help="Use syslog for logging.") - parser.add_option_group(group) - - -def setup_logging(options, conf): +def setup_logging(conf): """ Sets up the logging options for a log with supplied name - :param options: Mapping of typed option key/values - :param conf: Mapping of untyped key/values from config file + :param conf: a cfg.ConfOpts object """ - if options.get('log_config', None): + if conf.log_config: # Use a logging configuration file for all settings... - if os.path.exists(options['log_config']): - logging.config.fileConfig(options['log_config']) + if os.path.exists(conf.log_config): + logging.config.fileConfig(conf.log_config) return else: raise RuntimeError("Unable to locate specified logging " - "config file: %s" % options['log_config']) + "config file: %s" % conf.log_config) - # If either the CLI option or the conf value - # is True, we set to True - debug = options.get('debug') or \ - get_option(conf, 'debug', type='bool', default=False) - verbose = options.get('verbose') or \ - get_option(conf, 'verbose', type='bool', default=False) root_logger = logging.root - if debug: + if conf.debug: root_logger.setLevel(logging.DEBUG) - elif verbose: + elif conf.verbose: root_logger.setLevel(logging.INFO) else: root_logger.setLevel(logging.WARNING) - # Set log configuration from options... - # Note that we use a hard-coded log format in the options - # because of Paste.Deploy bug #379 - # http://trac.pythonpaste.org/pythonpaste/ticket/379 - log_format = options.get('log_format', DEFAULT_LOG_FORMAT) - log_date_format = options.get('log_date_format', DEFAULT_LOG_DATE_FORMAT) - formatter = logging.Formatter(log_format, log_date_format) + formatter = logging.Formatter(conf.log_format, conf.log_date_format) - logfile = options.get('log_file') - if not logfile: - logfile = conf.get('log_file') - - use_syslog = options.get('use_syslog') or \ - get_option(conf, 'use_syslog', type='bool', default=False) - - if use_syslog: + if conf.use_syslog: handler = logging.handlers.SysLogHandler(address='/dev/log') - elif logfile: - logdir = options.get('log_dir') - if not logdir: - logdir = conf.get('log_dir') - if logdir: - logfile = os.path.join(logdir, logfile) + elif conf.log_file: + logfile = conf.log_file + if conf.log_dir: + logfile = os.path.join(conf.log_dir, logfile) handler = logging.handlers.WatchedFileHandler(logfile) else: handler = logging.StreamHandler(sys.stdout) @@ -181,143 +90,39 @@ def setup_logging(options, conf): root_logger.addHandler(handler) -def find_config_file(conf_name, options, args): - """ - Return the first config file found for an application. - - We search for the paste config file in the following order: - * If --config-file option is used, use that - * If args[0] is a file, use that - * Search for $conf_name.conf in standard directories: - * ~.glance/ - * ~ - * /etc/glance - * /etc - - :retval Full path to config file. - - :raises RuntimeError if the config file cannot be found. - """ - - fix_path = lambda p: os.path.abspath(os.path.expanduser(p)) - if options.get('config_file'): - if os.path.exists(options['config_file']): - return fix_path(options['config_file']) - elif args: - if os.path.exists(args[0]): - return fix_path(args[0]) - - # Handle standard directory search for $conf_name.conf - config_file_dirs = [fix_path(os.path.join('~', '.glance')), - fix_path('~'), - '/etc/glance/', - '/etc'] - - for cfg_dir in config_file_dirs: - cfg_file = os.path.join(cfg_dir, '%s.conf' % conf_name) - if os.path.exists(cfg_file): - return cfg_file - - raise RuntimeError("Unable to locate %s configuration file." % conf_name) - - -def load_paste_config(conf_file, app_name): - """ - Load the configuration mapping from a paste config file. - - :param conf_file: The path to the paste config file. - :param app_name: Name of the application to load config for, or None. - None signifies to only load the [DEFAULT] section of - the config file. - :retval The configuration mapping. - - :raises RuntimeError when there was a problem loading the configuration - file. - """ - try: - return deploy.appconfig("config:%s" % conf_file, name=app_name) - except Exception, e: - raise RuntimeError("Error trying to load config %s: %s" - % (conf_file, e)) - - -def load_paste_app(app_name, options, args, conf_name=None): +def load_paste_app(conf, app_name=None): """ Builds and returns a WSGI app from a paste config file. - We search for the paste config file in the following order: - * If --config-file option is used, use that - * If args[0] is a file, use that - * Search for $conf_name.conf in standard directories: - * ~.glance/ - * ~ - * /etc/glance - * /etc + We assume the last config file specified in the supplied ConfigOpts + object is the paste config file. - :param app_name: Name of the application to load - :param options: Set of typed options returned from parse_options() - :param args: Command line arguments from argv[1:] - :param conf_name: Name of config file to load, defaults to app_name + :param conf: a cfg.ConfigOpts object + :param app_name: name of the application to load :raises RuntimeError when config file cannot be located or application cannot be loaded from config file """ - if conf_name is None: - conf_name = app_name - conf_file = find_config_file(conf_name, options, args) + if app_name is None: + app_name = conf.prog - conf = load_paste_config(conf_file, app_name) + # Assume paste config is in the last config file + conf_file = os.path.abspath(conf.config_file[-1]) try: - # Setup logging early, supplying both the CLI options and the - # configuration mapping from the config file - setup_logging(options, conf) + # Setup logging early + setup_logging(conf) - # We only update the conf dict for the verbose and debug - # flags. Everything else must be set up in the conf file... - debug = options.get('debug') or \ - get_option(conf, 'debug', type='bool', default=False) - verbose = options.get('verbose') or \ - get_option(conf, 'verbose', type='bool', default=False) - conf['debug'] = debug - conf['verbose'] = verbose + logger = logging.getLogger(app_name) + + app = wsgi.paste_deploy_app(conf_file, app_name, conf) # Log the options used when starting if we're in debug mode... - if debug: - logger = logging.getLogger(app_name) - logger.debug("*" * 80) - logger.debug("Configuration options gathered from config file:") - logger.debug(conf_file) - logger.debug("================================================") - items = dict([(k, v) for k, v in conf.items() - if k not in ('__file__', 'here')]) - for key, value in sorted(items.items()): - logger.debug("%(key)-30s %(value)s" % locals()) - logger.debug("*" * 80) - app = deploy.loadapp("config:%s" % conf_file, name=app_name) + if conf.debug: + conf.log_opt_values(logging.getLogger(app_name), logging.DEBUG) + + return app except (LookupError, ImportError), e: raise RuntimeError("Unable to load %(app_name)s from " "configuration file %(conf_file)s." "\nGot: %(e)r" % locals()) - return conf, app - - -def get_option(options, option, **kwargs): - if option in options: - value = options[option] - type_ = kwargs.get('type', 'str') - if type_ == 'bool': - if hasattr(value, 'lower'): - return value.lower() == 'true' - else: - return value - elif type_ == 'int': - return int(value) - elif type_ == 'float': - return float(value) - else: - return value - elif 'default' in kwargs: - return kwargs['default'] - else: - raise KeyError("option '%s' not found" % option) diff --git a/glance/common/context.py b/glance/common/context.py index e756540bdd..88f05dd5f5 100644 --- a/glance/common/context.py +++ b/glance/common/context.py @@ -14,7 +14,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -from glance.common import config + +from glance.common import cfg from glance.common import exception from glance.common import utils from glance.common import wsgi @@ -53,26 +54,29 @@ class RequestContext(object): class ContextMiddleware(wsgi.Middleware): - def __init__(self, app, conf): + + opts = [ + cfg.BoolOpt('owner_is_tenant', default=True), + ] + + def __init__(self, app, conf, **local_conf): self.conf = conf + self.conf.register_opts(self.opts) + + # Determine the context class to use + self.ctxcls = RequestContext + if 'context_class' in local_conf: + self.ctxcls = utils.import_class(local_conf['context_class']) + super(ContextMiddleware, self).__init__(app) def make_context(self, *args, **kwargs): """ Create a context with the given arguments. """ + kwargs.setdefault('owner_is_tenant', self.conf.owner_is_tenant) - # Determine the context class to use - ctxcls = RequestContext - if 'context_class' in self.conf: - ctxcls = utils.import_class(self.conf['context_class']) - - # Determine whether to use tenant or owner - owner_is_tenant = config.get_option(self.conf, 'owner_is_tenant', - type='bool', default=True) - kwargs.setdefault('owner_is_tenant', owner_is_tenant) - - return ctxcls(*args, **kwargs) + return self.ctxcls(*args, **kwargs) def process_request(self, req): """ diff --git a/glance/common/notifier.py b/glance/common/notifier.py index f2561693f6..5d5ed83396 100644 --- a/glance/common/notifier.py +++ b/glance/common/notifier.py @@ -22,7 +22,7 @@ import uuid import kombu.connection -from glance.common import config +from glance.common import cfg from glance.common import exception @@ -61,33 +61,29 @@ class LoggingStrategy(object): class RabbitStrategy(object): """A notifier that puts a message on a queue when called.""" + opts = [ + cfg.StrOpt('rabbit_host', default='localhost'), + cfg.IntOpt('rabbit_port', default=5672), + cfg.BoolOpt('rabbit_use_ssl', default=False), + cfg.StrOpt('rabbit_userid', default='guest'), + cfg.StrOpt('rabbit_password', default='guest'), + cfg.StrOpt('rabbit_virtual_host', default='/'), + cfg.StrOpt('rabbit_notification_topic', default='glance_notifications') + ] + def __init__(self, conf): """Initialize the rabbit notification strategy.""" self._conf = conf - host = self._get_option('rabbit_host', 'str', 'localhost') - port = self._get_option('rabbit_port', 'int', 5672) - use_ssl = self._get_option('rabbit_use_ssl', 'bool', False) - userid = self._get_option('rabbit_userid', 'str', 'guest') - password = self._get_option('rabbit_password', 'str', 'guest') - virtual_host = self._get_option('rabbit_virtual_host', 'str', '/') + self._conf.register_opts(self.opts) self.connection = kombu.connection.BrokerConnection( - hostname=host, - userid=userid, - password=password, - virtual_host=virtual_host, - ssl=use_ssl) + hostname=self._conf.rabbit_host, + userid=self._conf.rabbit_userid, + password=self._conf.rabbit_password, + virtual_host=self._conf.rabbit_virtual_host, + ssl=self._conf.rabbit_use_ssl) - self.topic = self._get_option('rabbit_notification_topic', - 'str', - 'glance_notifications') - - def _get_option(self, name, datatype, default): - """Retrieve a configuration option.""" - return config.get_option(self._conf, - name, - type=datatype, - default=default) + self.topic = self._conf.rabbit_notification_topic def _send_message(self, message, priority): topic = "%s.%s" % (self.topic, priority) @@ -115,9 +111,13 @@ class Notifier(object): "default": NoopStrategy, } + opts = [ + cfg.StrOpt('notifier_strategy', default='default') + ] + def __init__(self, conf, strategy=None): - strategy = config.get_option(conf, "notifier_strategy", - type="str", default="default") + conf.register_opts(self.opts) + strategy = conf.notifier_strategy try: self.strategy = self.STRATEGIES[strategy](conf) except KeyError: diff --git a/glance/common/wsgi.py b/glance/common/wsgi.py index 89fe0f1a70..f0055887c9 100644 --- a/glance/common/wsgi.py +++ b/glance/common/wsgi.py @@ -31,15 +31,29 @@ import time import eventlet from eventlet.green import socket, ssl import eventlet.wsgi +from paste import deploy import routes import routes.middleware import webob.dec import webob.exc +from glance.common import cfg from glance.common import exception from glance.common import utils +bind_opts = [ + cfg.StrOpt('bind_host', default='0.0.0.0'), + cfg.IntOpt('bind_port'), +] + +socket_opts = [ + cfg.IntOpt('backlog', default=4096), + cfg.StrOpt('cert_file'), + cfg.StrOpt('key_file'), + ] + + class WritableLogger(object): """A thin wrapper that responds to `write` and logs.""" @@ -51,30 +65,37 @@ class WritableLogger(object): self.logger.log(self.level, msg.strip("\n")) -def get_socket(host, port, conf): +def get_bind_addr(conf, default_port=None): + """Return the host and port to bind to.""" + conf.register_opts(bind_opts) + return (conf.bind_host, conf.bind_port or default_port) + + +def get_socket(conf, default_port): """ Bind socket to bind ip:port in conf note: Mostly comes from Swift with a few small changes... - :param host: Host to bind to - :param port: Port to bind to - :param conf: Configuration dict to read settings from + :param conf: a cfg.ConfigOpts object + :param default_port: port to bind to if none is specified in conf :returns : a socket object as returned from socket.listen or ssl.wrap_socket if conf specifies cert_file """ - bind_addr = (host, port) + bind_addr = get_bind_addr(conf, default_port) + # TODO(jaypipes): eventlet's greened socket module does not actually # support IPv6 in getaddrinfo(). We need to get around this in the # future or monitor upstream for a fix address_family = [addr[0] for addr in socket.getaddrinfo(bind_addr[0], bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM) if addr[0] in (socket.AF_INET, socket.AF_INET6)][0] - backlog = int(conf.get('backlog', 4096)) - cert_file = conf.get('cert_file') - key_file = conf.get('key_file') + conf.register_opts(socket_opts) + + cert_file = conf.cert_file + key_file = conf.key_file use_ssl = cert_file or key_file if use_ssl and (not cert_file or not key_file): raise RuntimeError(_("When running server in SSL mode, you must " @@ -85,7 +106,7 @@ def get_socket(host, port, conf): retry_until = time.time() + 30 while not sock and time.time() < retry_until: try: - sock = eventlet.listen(bind_addr, backlog=backlog, + sock = eventlet.listen(bind_addr, backlog=conf.backlog, family=address_family) if use_ssl: sock = ssl.wrap_socket(sock, certfile=cert_file, @@ -114,17 +135,15 @@ class Server(object): def __init__(self, threads=1000): self.pool = eventlet.GreenPool(threads) - def start(self, application, port, host='0.0.0.0', conf=None): + def start(self, application, conf, default_port): """ Run a WSGI server with the given application. :param application: The application to run in the WSGI server - :param port: Port to bind to - :param host: Host to bind to - :param conf: Mapping of configuration options + :param conf: a cfg.ConfigOpts object + :param default_port: Port to bind to if none is specified in conf """ - conf = conf or {} - socket = get_socket(host, port, conf) + socket = get_socket(conf, default_port) self.pool.spawn_n(self._run, application, socket) def wait(self): @@ -409,11 +428,45 @@ class Resource(object): return args -def _import_factory(conf, key): - return utils.import_class(conf[key].replace(':', '.').strip()) +class BasePasteFactory(object): + + """A base class for paste app and filter factories. + + Sub-classes must override the KEY class attribute and provide + a __call__ method. + """ + + KEY = None + + def __init__(self, conf): + self.conf = conf + + def __call__(self, global_conf, **local_conf): + raise NotImplementedError + + def _import_factory(self, local_conf): + """Import an app/filter class. + + Lookup the KEY from the PasteDeploy local conf and import the + class named there. This class can then be used as an app or + filter factory. + + Note we support the : format. + + Note also that if you do e.g. + + key = + value + + then ConfigParser returns a value with a leading newline, so + we strip() the value before using it. + """ + class_name = local_conf[self.KEY].replace(':', '.').strip() + return utils.import_class(class_name) -def app_factory(global_conf, **local_conf): +class AppFactory(BasePasteFactory): + """A Generic paste.deploy app factory. This requires glance.app_factory to be set to a callable which returns a @@ -423,18 +476,20 @@ def app_factory(global_conf, **local_conf): paste.app_factory = glance.common.wsgi:app_factory glance.app_factory = glance.api.v1:API - The WSGI app constructor must accept a configuration dict as its only - argument. + The WSGI app constructor must accept a ConfigOpts object and a local config + dict as its two arguments. """ - factory = _import_factory(local_conf, 'glance.app_factory') - conf = global_conf.copy() - conf.update(local_conf) + KEY = 'glance.app_factory' - return factory(conf) + def __call__(self, global_conf, **local_conf): + """The actual paste.app_factory protocol method.""" + factory = self._import_factory(local_conf) + return factory(self.conf, **local_conf) -def filter_factory(global_conf, **local_conf): +class FilterFactory(AppFactory): + """A Generic paste.deploy filter factory. This requires glance.filter_factory to be set to a callable which returns a @@ -444,15 +499,66 @@ def filter_factory(global_conf, **local_conf): paste.filter_factory = glance.common.wsgi:filter_factory glance.filter_factory = glance.api.middleware.cache:CacheFilter - The WSGI filter constructor must accept a WSGI app and a configuration dict - as its only two arguments. + The WSGI filter constructor must accept a WSGI app, a ConfigOpts object and + a local config dict as its three arguments. """ - factory = _import_factory(local_conf, 'glance.filter_factory') - conf = global_conf.copy() - conf.update(local_conf) + KEY = 'glance.filter_factory' - def filter(app): - return factory(app, conf) + def __call__(self, global_conf, **local_conf): + """The actual paste.filter_factory protocol method.""" + factory = self._import_factory(local_conf) - return filter + def filter(app): + return factory(app, self.conf, **local_conf) + + return filter + + +def setup_paste_factories(conf): + """Set up the generic paste app and filter factories. + + Set things up so that: + + paste.app_factory = glance.common.wsgi:app_factory + + and + + paste.filter_factory = glance.common.wsgi:filter_factory + + work correctly while loading PasteDeploy configuration. + + The app factories are constructed at runtime to allow us to pass a + ConfigOpts object to the WSGI classes. + + :param conf: a ConfigOpts object + """ + global app_factory, filter_factory + app_factory = AppFactory(conf) + filter_factory = FilterFactory(conf) + + +def teardown_paste_factories(): + """Reverse the effect of setup_paste_factories().""" + global app_factory, filter_factory + del app_factory + del filter_factory + + +def paste_deploy_app(paste_config_file, app_name, conf): + """Load a WSGI app from a PasteDeploy configuration. + + Use deploy.loadapp() to load the app from the PasteDeploy configuration, + ensuring that the supplied ConfigOpts object is passed to the app and + filter constructors. + + :param paste_config_file: a PasteDeploy config file + :param app_name: the name of the app/pipeline to load from the file + :param conf: a ConfigOpts object to supply to the app and its filters + :returns: the WSGI app + """ + setup_paste_factories(conf) + try: + return deploy.loadapp("config:%s" % paste_config_file, name=app_name) + finally: + teardown_paste_factories() diff --git a/glance/image_cache/__init__.py b/glance/image_cache/__init__.py index 543a2533fe..d792d57673 100644 --- a/glance/image_cache/__init__.py +++ b/glance/image_cache/__init__.py @@ -22,7 +22,7 @@ LRU Cache for Image Data import logging import os -from glance.common import config +from glance.common import cfg from glance.common import exception from glance.common import utils @@ -34,15 +34,23 @@ class ImageCache(object): """Provides an LRU cache for image data.""" + opts = [ + cfg.StrOpt('image_cache_driver', default='sqlite'), + cfg.IntOpt('image_cache_max_size', default=10 * (1024 ** 3)), # 10 GB + cfg.IntOpt('image_cache_stall_time', default=86400), # 24 hours + cfg.StrOpt('image_cache_dir'), + ] + def __init__(self, conf): self.conf = conf + self.conf.register_opts(self.opts) self.init_driver() def init_driver(self): """ Create the driver for the cache """ - driver_name = self.conf.get('image_cache_driver', 'sqlite') + driver_name = self.conf.image_cache_driver driver_module = (__name__ + '.drivers.' + driver_name + '.Driver') try: self.driver_class = utils.import_class(driver_module) @@ -150,8 +158,7 @@ class ImageCache(object): size. Returns a tuple containing the total number of cached files removed and the total size of all pruned image files. """ - max_size = int(self.conf.get('image_cache_max_size', - DEFAULT_MAX_CACHE_SIZE)) + max_size = self.conf.image_cache_max_size current_size = self.driver.get_cache_size() if max_size > current_size: logger.debug(_("Image cache has free space, skipping prune...")) @@ -180,12 +187,12 @@ class ImageCache(object): "%(total_bytes_pruned)d.") % locals()) return total_files_pruned, total_bytes_pruned - def clean(self): + def clean(self, stall_time=None): """ Cleans up any invalid or incomplete cached images. The cache driver decides what that means... """ - self.driver.clean() + self.driver.clean(stall_time) def queue_image(self, image_id): """ diff --git a/glance/image_cache/cleaner.py b/glance/image_cache/cleaner.py index f93c4d1b61..a6d8d91147 100644 --- a/glance/image_cache/cleaner.py +++ b/glance/image_cache/cleaner.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) class Cleaner(object): - def __init__(self, conf): + def __init__(self, conf, **local_conf): self.conf = conf self.cache = ImageCache(conf) diff --git a/glance/image_cache/drivers/base.py b/glance/image_cache/drivers/base.py index 4c28a44009..d2d0124bde 100644 --- a/glance/image_cache/drivers/base.py +++ b/glance/image_cache/drivers/base.py @@ -60,11 +60,9 @@ class Driver(object): Creates all necessary directories under the base cache directory """ - try: - key = 'image_cache_dir' - self.base_dir = self.conf[key] - except KeyError: - msg = _('Failed to read %s from config') % key + self.base_dir = self.conf.image_cache_dir + if self.base_dir is None: + msg = _('Failed to read %s from config') % 'image_cache_dir' logger.error(msg) driver = self.__class__.__module__ raise exception.BadDriverConfiguration(driver_name=driver, @@ -168,7 +166,7 @@ class Driver(object): :param image_id: Image ID """ - def clean(self): + def clean(self, stall_time=None): """ Dependent on the driver, clean up and destroy any invalid or incomplete cached images diff --git a/glance/image_cache/drivers/sqlite.py b/glance/image_cache/drivers/sqlite.py index 9d4c5b5e00..6f4a4ad961 100644 --- a/glance/image_cache/drivers/sqlite.py +++ b/glance/image_cache/drivers/sqlite.py @@ -31,13 +31,12 @@ import time from eventlet import sleep, timeout import sqlite3 +from glance.common import cfg from glance.common import exception from glance.image_cache.drivers import base logger = logging.getLogger(__name__) DEFAULT_SQL_CALL_TIMEOUT = 2 -DEFAULT_STALL_TIME = 86400 # 24 hours -DEFAULT_SQLITE_DB = 'cache.db' class SqliteConnection(sqlite3.Connection): @@ -82,6 +81,10 @@ class Driver(base.Driver): that has atimes set. """ + opts = [ + cfg.StrOpt('image_cache_sqlite_db', default='cache.db'), + ] + def configure(self): """ Configure the driver to use the stored configuration options @@ -91,11 +94,13 @@ class Driver(base.Driver): """ super(Driver, self).configure() + self.conf.register_opts(self.opts) + # Create the SQLite database that will hold our cache attributes self.initialize_db() def initialize_db(self): - db = self.conf.get('image_cache_sqlite_db', DEFAULT_SQLITE_DB) + db = self.conf.image_cache_sqlite_db self.db_path = os.path.join(self.base_dir, db) try: conn = sqlite3.connect(self.db_path, check_same_thread=False, @@ -244,7 +249,7 @@ class Driver(base.Driver): if os.path.exists(path): os.unlink(path) - def clean(self): + def clean(self, stall_time=None): """ Delete any image files in the invalid directory and any files in the incomplete directory that are older than a @@ -252,10 +257,11 @@ class Driver(base.Driver): """ self.delete_invalid_files() - incomplete_stall_time = int(self.conf.get('image_cache_stall_time', - DEFAULT_STALL_TIME)) + if stall_time is None: + stall_time = self.conf.image_cache_stall_time + now = time.time() - older_than = now - incomplete_stall_time + older_than = now - stall_time self.delete_stalled_files(older_than) def get_least_recently_accessed(self): diff --git a/glance/image_cache/drivers/xattr.py b/glance/image_cache/drivers/xattr.py index b1e30bd508..549cca7055 100644 --- a/glance/image_cache/drivers/xattr.py +++ b/glance/image_cache/drivers/xattr.py @@ -70,8 +70,6 @@ from glance.image_cache.drivers import base logger = logging.getLogger(__name__) -DEFAULT_STALL_TIME = 86400 # 24 hours - class Driver(base.Driver): @@ -416,7 +414,7 @@ class Driver(base.Driver): return self._reap_old_files(self.incomplete_dir, 'stalled', grace=grace) - def clean(self): + def clean(self, stall_time=None): """ Delete any image files in the invalid directory and any files in the incomplete directory that are older than a @@ -424,9 +422,10 @@ class Driver(base.Driver): """ self.reap_invalid() - incomplete_stall_time = int(self.conf.get('image_cache_stall_time', - DEFAULT_STALL_TIME)) - self.reap_stalled(incomplete_stall_time) + if stall_time is None: + stall_time = self.conf.image_cache_stall_time + + self.reap_stalled(stall_time) def get_all_regular_files(basepath): diff --git a/glance/image_cache/prefetcher.py b/glance/image_cache/prefetcher.py index ffdf6942c0..9e918d210e 100644 --- a/glance/image_cache/prefetcher.py +++ b/glance/image_cache/prefetcher.py @@ -23,8 +23,6 @@ import logging import eventlet -from glance.common import config -from glance.common import context from glance.common import exception from glance.image_cache import ImageCache from glance import registry @@ -42,16 +40,15 @@ logger = logging.getLogger(__name__) class Prefetcher(object): - def __init__(self, conf): + def __init__(self, conf, **local_conf): self.conf = conf glance.store.create_stores(conf) self.cache = ImageCache(conf) registry.configure_registry_client(conf) def fetch_image_into_cache(self, image_id): - auth_tok = self.conf.get('admin_token') - ctx = context.RequestContext(is_admin=True, show_deleted=True, - auth_tok=auth_tok) + ctx = registry.get_client_context(self.conf, + is_admin=True, show_deleted=True) try: image_meta = registry.get_image_metadata(ctx, image_id) if image_meta['status'] != 'active': diff --git a/glance/image_cache/pruner.py b/glance/image_cache/pruner.py index 14769c8dd2..4f43ea12e1 100644 --- a/glance/image_cache/pruner.py +++ b/glance/image_cache/pruner.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) class Pruner(object): - def __init__(self, conf): + def __init__(self, conf, **local_conf): self.conf = conf self.cache = ImageCache(conf) diff --git a/glance/image_cache/queue_image.py b/glance/image_cache/queue_image.py index 6718217198..2160351171 100644 --- a/glance/image_cache/queue_image.py +++ b/glance/image_cache/queue_image.py @@ -23,8 +23,6 @@ import logging import eventlet -from glance.common import config -from glance.common import context from glance.common import exception from glance.image_cache import ImageCache from glance import registry @@ -35,15 +33,14 @@ logger = logging.getLogger(__name__) class Queuer(object): - def __init__(self, conf): + def __init__(self, conf, **local_conf): self.conf = conf self.cache = ImageCache(conf) registry.configure_registry_client(conf) def queue_image(self, image_id): - auth_tok = self.conf.get('admin_token') - ctx = context.RequestContext(is_admin=True, show_deleted=True, - auth_tok=auth_tok) + ctx = \ + registry.get_client_context(conf, is_admin=True, show_deleted=True) try: image_meta = registry.get_image_metadata(ctx, image_id) if image_meta['status'] != 'active': diff --git a/glance/registry/__init__.py b/glance/registry/__init__.py index 04607b33a4..06db1db102 100644 --- a/glance/registry/__init__.py +++ b/glance/registry/__init__.py @@ -21,7 +21,7 @@ Registry API import logging -from glance.common import config +from glance.common import cfg from glance.common import exception from glance.registry import client @@ -34,6 +34,25 @@ _CLIENT_KWARGS = {} _METADATA_ENCRYPTION_KEY = None +registry_addr_opts = [ + cfg.StrOpt('registry_host', default='0.0.0.0'), + cfg.IntOpt('registry_port', default=9191), + ] +registry_client_opts = [ + cfg.StrOpt('registry_client_protocol', default='http'), + cfg.StrOpt('registry_client_key_file'), + cfg.StrOpt('registry_client_cert_file'), + cfg.StrOpt('registry_client_ca_file'), + cfg.StrOpt('metadata_encryption_key'), + ] +admin_token_opt = cfg.StrOpt('admin_token') + + +def get_registry_addr(conf): + conf.register_opts(registry_addr_opts) + return (conf.registry_host, conf.registry_port) + + def configure_registry_client(conf): """ Sets up a registry client for use in registry lookups @@ -42,9 +61,8 @@ def configure_registry_client(conf): """ global _CLIENT_KWARGS, _CLIENT_HOST, _CLIENT_PORT, _METADATA_ENCRYPTION_KEY try: - host = conf['registry_host'] - port = int(conf['registry_port']) - except (TypeError, ValueError): + host, port = get_registry_addr(conf) + except cfg.ConfigFileValueError: msg = _("Configuration option was not valid") logger.error(msg) raise exception.BadRegistryConnectionConfiguration(msg) @@ -53,18 +71,23 @@ def configure_registry_client(conf): logger.error(msg) raise exception.BadRegistryConnectionConfiguration(msg) - use_ssl = config.get_option(conf, 'registry_client_protocol', - default='http').lower() == 'https' - key_file = conf.get('registry_client_key_file') - cert_file = conf.get('registry_client_cert_file') - ca_file = conf.get('registry_client_ca_file') - _METADATA_ENCRYPTION_KEY = conf.get('metadata_encryption_key') + conf.register_opts(registry_client_opts) + _CLIENT_HOST = host _CLIENT_PORT = port - _CLIENT_KWARGS = {'use_ssl': use_ssl, - 'key_file': key_file, - 'cert_file': cert_file, - 'ca_file': ca_file} + _METADATA_ENCRYPTION_KEY = conf.metadata_encryption_key + _CLIENT_KWARGS = { + 'use_ssl': conf.registry_client_protocol.lower() == 'https', + 'key_file': conf.registry_client_key_file, + 'cert_file': conf.registry_client_cert_file, + 'ca_file': conf.registry_client_ca_file + } + + +def get_client_context(conf, **kwargs): + conf.register_opt(admin_token_opt) + from glance.common import context + return context.RequestContext(auth_tok=conf.admin_token, **kwargs) def get_registry_client(cxt): diff --git a/glance/registry/api/v1/__init__.py b/glance/registry/api/v1/__init__.py index a975429733..b840fd3fda 100644 --- a/glance/registry/api/v1/__init__.py +++ b/glance/registry/api/v1/__init__.py @@ -25,7 +25,7 @@ from glance.common import wsgi class API(wsgi.Router): """WSGI entry point for all Registry requests.""" - def __init__(self, conf): + def __init__(self, conf, **local_conf): mapper = routes.Mapper() images_resource = images.create_resource(conf) diff --git a/glance/registry/api/v1/images.py b/glance/registry/api/v1/images.py index fb0a247b7e..612038728f 100644 --- a/glance/registry/api/v1/images.py +++ b/glance/registry/api/v1/images.py @@ -23,6 +23,7 @@ import logging from webob import exc +from glance.common import cfg from glance.common import exception from glance.common import utils from glance.common import wsgi @@ -49,8 +50,14 @@ SUPPORTED_PARAMS = ('limit', 'marker', 'sort_key', 'sort_dir') class Controller(object): + opts = [ + cfg.IntOpt('limit_param_default', default=25), + cfg.IntOpt('api_limit_max', default=1000), + ] + def __init__(self, conf): self.conf = conf + self.conf.register_opts(self.opts) db_api.configure_db(conf) def _get_images(self, context, **params): @@ -181,31 +188,15 @@ class Controller(object): def _get_limit(self, req): """Parse a limit query param into something usable.""" try: - default = self.conf['limit_param_default'] - except KeyError: - # if no value is configured, provide a sane default - default = 25 - msg = _("Failed to read limit_param_default from config. " - "Defaulting to %s") % default - logger.debug(msg) - - try: - limit = int(req.str_params.get('limit', default)) + limit = int(req.str_params.get('limit', + self.conf.limit_param_default)) except ValueError: raise exc.HTTPBadRequest(_("limit param must be an integer")) if limit < 0: raise exc.HTTPBadRequest(_("limit param must be positive")) - try: - api_limit_max = int(self.conf['api_limit_max']) - except (KeyError, ValueError): - api_limit_max = 1000 - msg = _("Failed to read api_limit_max from config. " - "Defaulting to %s") % api_limit_max - logger.debug(msg) - - return min(api_limit_max, limit) + return min(self.conf.api_limit_max, limit) def _get_marker(self, req): """Parse a marker query param into something usable.""" diff --git a/glance/registry/db/__init__.py b/glance/registry/db/__init__.py index 7d20696617..96394fb82f 100644 --- a/glance/registry/db/__init__.py +++ b/glance/registry/db/__init__.py @@ -17,23 +17,24 @@ # License for the specific language governing permissions and limitations # under the License. -import optparse +from glance.common import cfg -def add_conf(parser): +def add_options(conf): """ Adds any configuration options that the db layer might have. - :param parser: An optparse.OptionParser object + :param conf: A ConfigOpts object :retval None """ - help_text = "The following configuration options are specific to the "\ - "Glance image registry database." - - group = optparse.OptionGroup(parser, "Registry Database Options", - help_text) - group.add_option('--sql-connection', metavar="CONNECTION", - default=None, - help="A valid SQLAlchemy connection string for the " - "registry database. Default: %default") - parser.add_option_group(group) + conf.add_group(cfg.OptGroup('registrydb', + title='Registry Database Options', + help='The following configuration options ' + 'are specific to the Glance image ' + 'registry database.')) + conf.register_cli_opt(cfg.StrOpt('sql-connection', + group='registrydb', + metavar='CONNECTION', + help='A valid SQLAlchemy connection ' + 'string for the registry database. ' + 'Default: %default')) diff --git a/glance/registry/db/api.py b/glance/registry/db/api.py index 2cce4cbcf3..13b54c97af 100644 --- a/glance/registry/db/api.py +++ b/glance/registry/db/api.py @@ -31,7 +31,7 @@ from sqlalchemy.orm import joinedload from sqlalchemy.orm import sessionmaker from sqlalchemy.sql import or_, and_ -from glance.common import config +from glance.common import cfg from glance.common import exception from glance.common import utils from glance.registry.db import models @@ -57,6 +57,11 @@ DISK_FORMATS = ['ami', 'ari', 'aki', 'vhd', 'vmdk', 'raw', 'qcow2', 'vdi', STATUSES = ['active', 'saving', 'queued', 'killed', 'pending_delete', 'deleted'] +db_opts = [ + cfg.IntOpt('sql_idle_timeout', default=3600), + cfg.StrOpt('sql_connection', default='sqlite:///glance.sqlite'), + ] + def configure_db(conf): """ @@ -67,13 +72,9 @@ def configure_db(conf): """ global _ENGINE, sa_logger, logger if not _ENGINE: - debug = config.get_option( - conf, 'debug', type='bool', default=False) - verbose = config.get_option( - conf, 'verbose', type='bool', default=False) - timeout = config.get_option( - conf, 'sql_idle_timeout', type='int', default=3600) - sql_connection = config.get_option(conf, 'sql_connection') + conf.register_opts(db_opts) + timeout = conf.sql_idle_timeout + sql_connection = conf.sql_connection try: _ENGINE = create_engine(sql_connection, pool_recycle=timeout) except Exception, err: @@ -84,9 +85,9 @@ def configure_db(conf): raise sa_logger = logging.getLogger('sqlalchemy.engine') - if debug: + if conf.debug: sa_logger.setLevel(logging.DEBUG) - elif verbose: + elif conf.verbose: sa_logger.setLevel(logging.INFO) models.register_models(_ENGINE) diff --git a/glance/registry/db/migration.py b/glance/registry/db/migration.py index e1ec9e6939..ee31eb13ae 100644 --- a/glance/registry/db/migration.py +++ b/glance/registry/db/migration.py @@ -40,7 +40,7 @@ def db_version(conf): :retval version number """ repo_path = get_migrate_repo_path() - sql_connection = conf['sql_connection'] + sql_connection = conf.sql_connection try: return versioning_api.db_version(sql_connection, repo_path) except versioning_exceptions.DatabaseNotControlledError, e: @@ -59,7 +59,7 @@ def upgrade(conf, version=None): """ db_version(conf) # Ensure db is under migration control repo_path = get_migrate_repo_path() - sql_connection = conf['sql_connection'] + sql_connection = conf.sql_connection version_str = version or 'latest' logger.info(_("Upgrading %(sql_connection)s to version %(version_str)s") % locals()) @@ -76,7 +76,7 @@ def downgrade(conf, version): """ db_version(conf) # Ensure db is under migration control repo_path = get_migrate_repo_path() - sql_connection = conf['sql_connection'] + sql_connection = conf.sql_connection logger.info(_("Downgrading %(sql_connection)s to version %(version)s") % locals()) return versioning_api.downgrade(sql_connection, repo_path, version) @@ -88,7 +88,7 @@ def version_control(conf): :param conf: conf dict """ - sql_connection = conf['sql_connection'] + sql_connection = conf.sql_connection try: _version_control(conf) except versioning_exceptions.DatabaseAlreadyControlledError, e: @@ -104,7 +104,7 @@ def _version_control(conf): :param conf: conf dict """ repo_path = get_migrate_repo_path() - sql_connection = conf['sql_connection'] + sql_connection = conf.sql_connection return versioning_api.version_control(sql_connection, repo_path) diff --git a/glance/store/__init__.py b/glance/store/__init__.py index 99f6918cc8..233ad1fbf5 100644 --- a/glance/store/__init__.py +++ b/glance/store/__init__.py @@ -22,7 +22,7 @@ import time import urlparse from glance import registry -from glance.common import config +from glance.common import cfg from glance.common import exception from glance.common import utils from glance.store import location @@ -154,13 +154,27 @@ def get_store_from_location(uri): return loc.store_name +scrubber_datadir_opt = cfg.StrOpt('scrubber_datadir', + default='/var/lib/glance/scrubber') + + +def get_scrubber_datadir(conf): + conf.register_opt(scrubber_datadir_opt) + return conf.scrubber_datadir + + +delete_opts = [ + cfg.BoolOpt('delayed_delete', default=False), + cfg.IntOpt('scrub_time', default=0) + ] + + def schedule_delete_from_backend(uri, conf, context, image_id, **kwargs): """ Given a uri and a time, schedule the deletion of an image. """ - use_delay = config.get_option(conf, 'delayed_delete', type='bool', - default=False) - if not use_delay: + conf.register_opts(delete_opts) + if not conf.delayed_delete: registry.update_image_metadata(context, image_id, {'status': 'deleted'}) try: @@ -169,10 +183,8 @@ def schedule_delete_from_backend(uri, conf, context, image_id, **kwargs): msg = _("Failed to delete image from store (%(uri)s).") % locals() logger.error(msg) - datadir = config.get_option(conf, 'scrubber_datadir') - scrub_time = config.get_option(conf, 'scrub_time', type='int', - default=0) - delete_time = time.time() + scrub_time + datadir = get_scrubber_datadir(conf) + delete_time = time.time() + conf.scrub_time file_path = os.path.join(datadir, str(image_id)) utils.safe_mkdirs(datadir) diff --git a/glance/store/base.py b/glance/store/base.py index 64b1c9896c..764ec7cd1b 100644 --- a/glance/store/base.py +++ b/glance/store/base.py @@ -28,13 +28,13 @@ class Store(object): CHUNKSIZE = (16 * 1024 * 1024) # 16M - def __init__(self, conf=None): + def __init__(self, conf): """ Initialize the Store :param conf: Optional dictionary of configuration options """ - self.conf = conf or {} + self.conf = conf self.configure() diff --git a/glance/store/filesystem.py b/glance/store/filesystem.py index 805d5a7766..fa8e24c05f 100644 --- a/glance/store/filesystem.py +++ b/glance/store/filesystem.py @@ -24,6 +24,7 @@ import logging import os import urlparse +from glance.common import cfg from glance.common import exception import glance.store import glance.store.base @@ -93,6 +94,8 @@ class ChunkedFile(object): class Store(glance.store.base.Store): + datadir_opt = cfg.StrOpt('filesystem_store_datadir') + def configure_add(self): """ Configure the Store to use the stored configuration options @@ -100,7 +103,15 @@ class Store(glance.store.base.Store): this method. If the store was not able to successfully configure itself, it should raise `exception.BadStoreConfiguration` """ - self.datadir = self._option_get('filesystem_store_datadir') + self.conf.register_opt(self.datadir_opt) + + self.datadir = self.conf.filesystem_store_datadir + if self.datadir is None: + reason = _("Could not find %s in configuration options.") % \ + 'filesystem_store_datadir' + logger.error(reason) + raise exception.BadStoreConfiguration(store_name="filesystem", + reason=reason) if not os.path.exists(self.datadir): msg = _("Directory to write image files does not exist " @@ -114,15 +125,6 @@ class Store(glance.store.base.Store): raise exception.BadStoreConfiguration(store_name="filesystem", reason=reason) - def _option_get(self, param): - result = self.conf.get(param) - if not result: - reason = _("Could not find %s in configuration options.") % param - logger.error(reason) - raise exception.BadStoreConfiguration(store_name="filesystem", - reason=reason) - return result - def get(self, location): """ Takes a `glance.store.location.Location` object that indicates diff --git a/glance/store/rbd.py b/glance/store/rbd.py index 465141cff5..27dd22d6ab 100644 --- a/glance/store/rbd.py +++ b/glance/store/rbd.py @@ -24,6 +24,7 @@ import hashlib import logging import math +from glance.common import cfg from glance.common import exception import glance.store import glance.store.base @@ -100,6 +101,13 @@ class Store(glance.store.base.Store): EXAMPLE_URL = "rbd://" + opts = [ + cfg.IntOpt('rbd_store_chunk_size', default=DEFAULT_CHUNKSIZE), + cfg.StrOpt('rbd_store_pool', default=DEFAULT_POOL), + cfg.StrOpt('rbd_store_user', default=DEFAULT_USER), + cfg.StrOpt('rbd_store_ceph_conf', default=DEFAULT_CONFFILE), + ] + def configure_add(self): """ Configure the Store to use the stored configuration options @@ -107,20 +115,16 @@ class Store(glance.store.base.Store): this method. If the store was not able to successfully configure itself, it should raise `exception.BadStoreConfiguration` """ + self.conf.register_opts(self.opts) try: - self.chunk_size = int( - self.conf.get( - 'rbd_store_chunk_size', - DEFAULT_CHUNKSIZE)) * 1024 * 1024 + self.chunk_size = self.conf.rbd_store_chunk_size * 1024 * 1024 + # these must not be unicode since they will be passed to a # non-unicode-aware C library - self.pool = str(self.conf.get('rbd_store_pool', - DEFAULT_POOL)) - self.user = str(self.conf.get('rbd_store_user', - DEFAULT_USER)) - self.conf_file = str(self.conf.get('rbd_store_ceph_conf', - DEFAULT_CONFFILE)) - except Exception, e: + self.pool = str(self.conf.rbd_store_pool) + self.user = str(self.conf.rbd_store_user) + self.conf_file = str(self.conf.rbd_store_ceph_conf) + except cfg.ConfigFileValueError, e: reason = _("Error in store configuration: %s") % e logger.error(reason) raise exception.BadStoreConfiguration(store_name='rbd', diff --git a/glance/store/s3.py b/glance/store/s3.py index 7edac44fee..9e8ab7f675 100644 --- a/glance/store/s3.py +++ b/glance/store/s3.py @@ -23,7 +23,7 @@ import httplib import tempfile import urlparse -from glance.common import config +from glance.common import cfg from glance.common import exception import glance.store import glance.store.base @@ -183,6 +183,15 @@ class Store(glance.store.base.Store): EXAMPLE_URL = "s3://:@//" + opts = [ + cfg.StrOpt('s3_store_host'), + cfg.StrOpt('s3_store_access_key'), + cfg.StrOpt('s3_store_secret_key'), + cfg.StrOpt('s3_store_bucket'), + cfg.StrOpt('s3_store_object_buffer_dir'), + cfg.BoolOpt('s3_store_create_bucket_on_put', default=False), + ] + def configure_add(self): """ Configure the Store to use the stored configuration options @@ -190,6 +199,7 @@ class Store(glance.store.base.Store): this method. If the store was not able to successfully configure itself, it should raise `exception.BadStoreConfiguration` """ + self.conf.register_opts(self.opts) self.s3_host = self._option_get('s3_store_host') access_key = self._option_get('s3_store_access_key') secret_key = self._option_get('s3_store_secret_key') @@ -210,14 +220,11 @@ class Store(glance.store.base.Store): else: # Defaults http self.full_s3_host = 'http://' + self.s3_host - if self.conf.get('s3_store_object_buffer_dir'): - self.s3_store_object_buffer_dir = self.conf.get( - 's3_store_object_buffer_dir') - else: - self.s3_store_object_buffer_dir = None + self.s3_store_object_buffer_dir = \ + self.conf.s3_store_object_buffer_dir def _option_get(self, param): - result = self.conf.get(param) + result = getattr(self.conf, param) if not result: reason = _("Could not find %(param)s in configuration " "options.") % locals() @@ -417,10 +424,7 @@ def create_bucket_if_missing(bucket, s3_conn, conf): s3_conn.get_bucket(bucket) except S3ResponseError, e: if e.status == httplib.NOT_FOUND: - add_bucket = config.get_option(conf, - 's3_store_create_bucket_on_put', - type='bool', default=False) - if add_bucket: + if conf.s3_store_create_bucket_on_put: try: s3_conn.create_bucket(bucket) except S3ResponseError, e: diff --git a/glance/store/scrubber.py b/glance/store/scrubber.py index e61e6e88ca..b1e227b396 100644 --- a/glance/store/scrubber.py +++ b/glance/store/scrubber.py @@ -27,7 +27,7 @@ import glance.store.s3 import glance.store.swift from glance import registry from glance import store -from glance.common import config +from glance.common import cfg from glance.common import utils from glance.common import exception from glance.registry import context @@ -65,22 +65,30 @@ class Daemon(object): class Scrubber(object): CLEANUP_FILE = ".cleanup" - def __init__(self, conf): - logger.info(_("Initializing scrubber with conf: %s") % conf) + opts = [ + cfg.BoolOpt('cleanup_scrubber', default=False), + cfg.IntOpt('cleanup_scrubber_time', default=86400) + ] + + def __init__(self, conf, **local_conf): self.conf = conf - self.datadir = config.get_option(conf, 'scrubber_datadir') - self.cleanup = config.get_option(conf, 'cleanup_scrubber', - type='bool', default=False) - host = config.get_option(conf, 'registry_host') - port = config.get_option(conf, 'registry_port', type='int') + self.conf.register_opts(self.opts) + + self.datadir = store.get_scrubber_datadir(conf) + self.cleanup = self.conf.cleanup_scrubber + self.cleanup_time = self.conf.cleanup_scrubber_time + + host, port = registry.get_registry_addr(conf) + + logger.info(_("Initializing scrubber with conf: %s") % + {'datadir': self.datadir, 'cleanup': self.cleanup, + 'cleanup_time': self.cleanup_time, + 'registry_host': host, 'registry_port': port}) + self.registry = client.RegistryClient(host, port) utils.safe_mkdirs(self.datadir) - if self.cleanup: - self.cleanup_time = config.get_option(conf, - 'cleanup_scrubber_time', - type='int', default=86400) store.create_stores(conf) def run(self, pool, event=None): diff --git a/glance/store/swift.py b/glance/store/swift.py index abe915ebfc..dc0dcad83c 100644 --- a/glance/store/swift.py +++ b/glance/store/swift.py @@ -26,7 +26,7 @@ import math import tempfile import urlparse -from glance.common import config +from glance.common import cfg from glance.common import exception import glance.store import glance.store.base @@ -38,8 +38,8 @@ except ImportError: pass DEFAULT_CONTAINER = 'glance' -DEFAULT_LARGE_OBJECT_SIZE = 5 * 1024 * 1024 * 1024 # 5GB -DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 200 * 1024 * 1024 # 200M +DEFAULT_LARGE_OBJECT_SIZE = 5 * 1024 # 5GB +DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 200 # 200M logger = logging.getLogger('glance.store.swift') @@ -185,9 +185,24 @@ class Store(glance.store.base.Store): CHUNKSIZE = 65536 + opts = [ + cfg.BoolOpt('swift_enable_snet', default=False), + cfg.StrOpt('swift_store_auth_address'), + cfg.StrOpt('swift_store_user'), + cfg.StrOpt('swift_store_key'), + cfg.StrOpt('swift_store_container', + default=DEFAULT_CONTAINER), + cfg.IntOpt('swift_store_large_object_size', + default=DEFAULT_LARGE_OBJECT_SIZE), + cfg.IntOpt('swift_store_large_object_chunk_size', + default=DEFAULT_LARGE_OBJECT_CHUNK_SIZE), + cfg.StrOpt('swift_store_object_buffer_dir'), + cfg.BoolOpt('swift_store_create_container_on_put', default=False), + ] + def configure(self): - self.snet = config.get_option( - self.conf, 'swift_enable_snet', type='bool', default=False) + self.conf.register_opts(self.opts) + self.snet = self.conf.swift_enable_snet def configure_add(self): """ @@ -199,29 +214,14 @@ class Store(glance.store.base.Store): self.auth_address = self._option_get('swift_store_auth_address') self.user = self._option_get('swift_store_user') self.key = self._option_get('swift_store_key') - self.container = self.conf.get('swift_store_container', - DEFAULT_CONTAINER) + self.container = self.conf.swift_store_container try: - if self.conf.get('swift_store_large_object_size'): - self.large_object_size = int( - self.conf.get('swift_store_large_object_size') - ) * (1024 * 1024) # Size specified in MB in conf files - else: - self.large_object_size = DEFAULT_LARGE_OBJECT_SIZE - - if self.conf.get('swift_store_large_object_chunk_size'): - self.large_object_chunk_size = int( - self.conf.get('swift_store_large_object_chunk_size') - ) * (1024 * 1024) # Size specified in MB in conf files - else: - self.large_object_chunk_size = DEFAULT_LARGE_OBJECT_CHUNK_SIZE - - if self.conf.get('swift_store_object_buffer_dir'): - self.swift_store_object_buffer_dir = ( - self.conf.get('swift_store_object_buffer_dir')) - else: - self.swift_store_object_buffer_dir = None - except Exception, e: + self.large_object_size = self.conf.swift_store_large_object_size + self.large_object_chunk_size = \ + self.conf.swift_store_large_object_chunk_size + self.swift_store_object_buffer_dir = \ + self.conf.swift_store_object_buffer_dir + except cfg.ConfigFileValueError, e: reason = _("Error in configuration conf: %s") % e logger.error(reason) raise exception.BadStoreConfiguration(store_name="swift", @@ -283,7 +283,7 @@ class Store(glance.store.base.Store): authurl=auth_url, user=user, key=key, snet=snet) def _option_get(self, param): - result = self.conf.get(param) + result = getattr(self.conf, param) if not result: reason = (_("Could not find %(param)s in configuration " "options.") % locals()) @@ -495,10 +495,7 @@ def create_container_if_missing(container, swift_conn, conf): swift_conn.head_container(container) except swift_client.ClientException, e: if e.http_status == httplib.NOT_FOUND: - add_container = config.get_option(conf, - 'swift_store_create_container_on_put', - type='bool', default=False) - if add_container: + if conf.swift_store_create_container_on_put: try: swift_conn.put_container(container) except ClientException, e: diff --git a/glance/tests/functional/test_bin_glance_cache_manage.py b/glance/tests/functional/test_bin_glance_cache_manage.py index 999f14e3ff..24dcd7f0ac 100644 --- a/glance/tests/functional/test_bin_glance_cache_manage.py +++ b/glance/tests/functional/test_bin_glance_cache_manage.py @@ -226,7 +226,8 @@ glance.app_factory = glance.image_cache.queue_image:Queuer """ % cache_file_options) cache_file.flush() - cmd = "bin/glance-cache-prefetcher %s" % cache_config_filepath + cmd = "bin/glance-cache-prefetcher --config-file %s" % \ + cache_config_filepath exitcode, out, err = execute(cmd) diff --git a/glance/tests/functional/test_cache_middleware.py b/glance/tests/functional/test_cache_middleware.py index b06450e8ad..c13cf702d3 100644 --- a/glance/tests/functional/test_cache_middleware.py +++ b/glance/tests/functional/test_cache_middleware.py @@ -374,7 +374,8 @@ glance.app_factory = glance.image_cache.queue_image:Queuer self.verify_no_cached_images() - cmd = "bin/glance-cache-prefetcher %s" % cache_config_filepath + cmd = "bin/glance-cache-prefetcher --config-file %s" % \ + cache_config_filepath exitcode, out, err = execute(cmd) diff --git a/glance/tests/functional/test_client_redirects.py b/glance/tests/functional/test_client_redirects.py index 8e0c022f34..13a0fa1ee1 100644 --- a/glance/tests/functional/test_client_redirects.py +++ b/glance/tests/functional/test_client_redirects.py @@ -86,8 +86,9 @@ class TestClientRedirects(functional.FunctionalTest): self.port_two = utils.get_unused_port() server_one = wsgi.Server() server_two = wsgi.Server() - server_one.start(RedirectTestApp("one"), self.port_one, "127.0.0.1") - server_two.start(RedirectTestApp("two"), self.port_two, "127.0.0.1") + conf = utils.TestConfigOpts({'bind_host': '127.0.0.1'}) + server_one.start(RedirectTestApp("one"), conf, self.port_one) + server_two.start(RedirectTestApp("two"), conf, self.port_two) self.client = client.BaseClient("127.0.0.1", self.port_one) def test_get_without_redirect(self): diff --git a/glance/tests/stubs.py b/glance/tests/stubs.py index 1abcfc07d1..8f8298fb4b 100644 --- a/glance/tests/stubs.py +++ b/glance/tests/stubs.py @@ -28,6 +28,7 @@ import glance.common.client from glance.common import context from glance.common import exception from glance.registry.api import v1 as rserver +from glance.tests import utils FAKE_FILESYSTEM_ROOTDIR = os.path.join('/tmp', 'glance-tests') @@ -97,9 +98,13 @@ def stub_out_registry_and_store_server(stubs): sql_connection = os.environ.get('GLANCE_SQL_CONNECTION', "sqlite://") context_class = 'glance.registry.context.RequestContext' - conf = {'sql_connection': sql_connection, 'verbose': VERBOSE, - 'debug': DEBUG, 'context_class': context_class} - api = context.ContextMiddleware(rserver.API(conf), conf) + conf = utils.TestConfigOpts({ + 'sql_connection': sql_connection, + 'verbose': VERBOSE, + 'debug': DEBUG + }) + api = context.ContextMiddleware(rserver.API(conf), + conf, context_class=context_class) res = self.req.get_response(api) # httplib.Response has a read() method...fake it out @@ -145,14 +150,16 @@ def stub_out_registry_and_store_server(stubs): self.req.body = body def getresponse(self): - conf = {'verbose': VERBOSE, + conf = utils.TestConfigOpts({ + 'verbose': VERBOSE, 'debug': DEBUG, 'bind_host': '0.0.0.0', 'bind_port': '9999999', 'registry_host': '0.0.0.0', 'registry_port': '9191', 'default_store': 'file', - 'filesystem_store_datadir': FAKE_FILESYSTEM_ROOTDIR} + 'filesystem_store_datadir': FAKE_FILESYSTEM_ROOTDIR + }) api = version_negotiation.VersionNegotiationFilter( context.ContextMiddleware(router.API(conf), conf), conf) @@ -218,9 +225,13 @@ def stub_out_registry_server(stubs, **kwargs): def getresponse(self): sql_connection = kwargs.get('sql_connection', "sqlite:///") context_class = 'glance.registry.context.RequestContext' - conf = {'sql_connection': sql_connection, 'verbose': VERBOSE, - 'debug': DEBUG, 'context_class': context_class} - api = context.ContextMiddleware(rserver.API(conf), conf) + conf = utils.TestConfigOpts({ + 'sql_connection': sql_connection, + 'verbose': VERBOSE, + 'debug': DEBUG + }) + api = context.ContextMiddleware(rserver.API(conf), + conf, context_class=context_class) res = self.req.get_response(api) # httplib.Response has a read() method...fake it out diff --git a/glance/tests/unit/test_api.py b/glance/tests/unit/test_api.py index 69b4472c23..3c824bd03f 100644 --- a/glance/tests/unit/test_api.py +++ b/glance/tests/unit/test_api.py @@ -34,6 +34,7 @@ from glance.registry.api import v1 as rserver from glance.registry.db import api as db_api from glance.registry.db import models as db_models from glance.tests import stubs +from glance.tests import utils as test_utils _gen_uuid = utils.generate_uuid @@ -48,8 +49,7 @@ CONF = {'sql_connection': 'sqlite://', 'registry_host': '0.0.0.0', 'registry_port': '9191', 'default_store': 'file', - 'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR, - 'context_class': 'glance.registry.context.RequestContext'} + 'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR} class TestRegistryDb(unittest.TestCase): @@ -64,9 +64,11 @@ class TestRegistryDb(unittest.TestCase): API controller results in a) an Exception being thrown and b) a message being logged to the registry log file... """ - bad_conf = {'verbose': True, - 'debug': True, - 'sql_connection': 'baddriver:///'} + bad_conf = test_utils.TestConfigOpts({ + 'verbose': True, + 'debug': True, + 'sql_connection': 'baddriver:///' + }) # We set this to None to trigger a reconfigure, otherwise # other modules may have already correctly configured the DB orig_engine = db_api._ENGINE @@ -101,7 +103,10 @@ class TestRegistryAPI(unittest.TestCase): self.stubs = stubout.StubOutForTesting() stubs.stub_out_registry_and_store_server(self.stubs) stubs.stub_out_filesystem_backend() - self.api = context.ContextMiddleware(rserver.API(CONF), CONF) + conf = test_utils.TestConfigOpts(CONF) + context_class = 'glance.registry.context.RequestContext' + self.api = context.ContextMiddleware(rserver.API(conf), + conf, context_class=context_class) self.FIXTURES = [ {'id': UUID1, 'name': 'fake image #1', @@ -136,7 +141,7 @@ class TestRegistryAPI(unittest.TestCase): 'location': "file:///tmp/glance-tests/2", 'properties': {}}] self.context = rcontext.RequestContext(is_admin=True) - db_api.configure_db(CONF) + db_api.configure_db(conf) self.destroy_fixtures() self.create_fixtures() @@ -1935,7 +1940,8 @@ class TestGlanceAPI(unittest.TestCase): stubs.stub_out_registry_and_store_server(self.stubs) stubs.stub_out_filesystem_backend() sql_connection = os.environ.get('GLANCE_SQL_CONNECTION', "sqlite://") - self.api = context.ContextMiddleware(router.API(CONF), CONF) + conf = test_utils.TestConfigOpts(CONF) + self.api = context.ContextMiddleware(router.API(conf), conf) self.FIXTURES = [ {'id': UUID1, 'name': 'fake image #1', @@ -1966,7 +1972,7 @@ class TestGlanceAPI(unittest.TestCase): 'location': "file:///tmp/glance-tests/2", 'properties': {}}] self.context = rcontext.RequestContext(is_admin=True) - db_api.configure_db(CONF) + db_api.configure_db(conf) self.destroy_fixtures() self.create_fixtures() diff --git a/glance/tests/unit/test_clients.py b/glance/tests/unit/test_clients.py index 8361cc8b41..aade2fd12b 100644 --- a/glance/tests/unit/test_clients.py +++ b/glance/tests/unit/test_clients.py @@ -34,6 +34,7 @@ from glance.registry.db import models as db_models from glance.registry import client as rclient from glance.registry import context as rcontext from glance.tests import stubs +from glance.tests import utils as test_utils CONF = {'sql_connection': 'sqlite://'} @@ -138,7 +139,8 @@ class TestRegistryClient(unittest.TestCase): """Establish a clean test environment""" self.stubs = stubout.StubOutForTesting() stubs.stub_out_registry_and_store_server(self.stubs) - db_api.configure_db(CONF) + conf = test_utils.TestConfigOpts(CONF) + db_api.configure_db(conf) self.context = rcontext.RequestContext(is_admin=True) self.FIXTURES = [ {'id': UUID1, @@ -1138,7 +1140,8 @@ class TestClient(unittest.TestCase): self.stubs = stubout.StubOutForTesting() stubs.stub_out_registry_and_store_server(self.stubs) stubs.stub_out_filesystem_backend() - db_api.configure_db(CONF) + conf = test_utils.TestConfigOpts(CONF) + db_api.configure_db(conf) self.client = client.Client("0.0.0.0") self.FIXTURES = [ {'id': UUID1, diff --git a/glance/tests/unit/test_config.py b/glance/tests/unit/test_config.py index d85819f957..56eb98c6ee 100644 --- a/glance/tests/unit/test_config.py +++ b/glance/tests/unit/test_config.py @@ -16,8 +16,6 @@ # under the License. import os.path -import optparse -import tempfile import unittest import stubout @@ -26,125 +24,9 @@ from glance.api.middleware import version_negotiation from glance.api.v1 import images from glance.api.v1 import members from glance.common import config -from glance.common import wsgi from glance.image_cache import pruner -class TestOptionParsing(unittest.TestCase): - - def test_common_options(self): - parser = optparse.OptionParser() - self.assertEquals(0, len(parser.option_groups)) - config.add_common_options(parser) - self.assertEquals(1, len(parser.option_groups)) - - expected_options = ['--verbose', '--debug', '--config-file'] - for e in expected_options: - self.assertTrue(parser.option_groups[0].get_option(e), - "Missing required common option: %s" % e) - - def test_parse_options(self): - # test empty args and that parse_options() returns a mapping - # of typed values - parser = optparse.OptionParser() - config.add_common_options(parser) - parsed_conf, args = config.parse_options(parser, []) - - expected_conf = {'verbose': False, 'debug': False, - 'config_file': None} - self.assertEquals(expected_conf, parsed_conf) - - # test non-empty args and that parse_options() returns a mapping - # of typed values matching supplied args - parser = optparse.OptionParser() - config.add_common_options(parser) - parsed_conf, args = config.parse_options(parser, ['--verbose']) - - expected_conf = {'verbose': True, 'debug': False, - 'config_file': None} - self.assertEquals(expected_conf, parsed_conf) - - # test non-empty args that contain unknown options raises - # a SystemExit exception. Not ideal, but unfortunately optparse - # raises a sys.exit() when it runs into an error instead of raising - # something a bit more useful for libraries. optparse must have been - # written by the same group that wrote unittest ;) - parser = optparse.OptionParser() - config.add_common_options(parser) - self.assertRaises(SystemExit, config.parse_options, - parser, ['--unknown']) - - -class TestConfigFiles(unittest.TestCase): - - def setUp(self): - self.stubs = stubout.StubOutForTesting() - - def tearDown(self): - self.stubs.UnsetAll() - - def test_config_file_default(self): - expected_path = '/etc/glance/glance-api.conf' - - self.stubs.Set(os.path, 'exists', lambda p: p == expected_path) - - path = config.find_config_file('glance-api', {}, []) - - self.assertEquals(expected_path, path) - - def test_config_file_option(self): - expected_path = '/etc/glance/my-glance-api.conf' - - self.stubs.Set(os.path, 'exists', lambda p: p == expected_path) - - path = config.find_config_file('glance-api', - {'config_file': expected_path}, []) - - self.assertEquals(expected_path, path) - - def test_config_file_arg(self): - expected_path = '/etc/glance/my-glance-api.conf' - - self.stubs.Set(os.path, 'exists', lambda p: p == expected_path) - - path = config.find_config_file('glance-api', {}, [expected_path]) - - self.assertEquals(expected_path, path) - - def test_config_file_tilde_arg(self): - supplied_path = '~/my-glance-api.conf' - expected_path = '/tmp/my-glance-api.conf' - - def fake_expanduser(p): - if p[0] == '~': - p = '/tmp' + p[1:] - return p - - self.stubs.Set(os.path, 'expanduser', fake_expanduser) - self.stubs.Set(os.path, 'exists', lambda p: p == supplied_path) - - path = config.find_config_file('glance-api', {}, [supplied_path]) - - self.assertEquals(expected_path, path) - - def test_config_file_not_found(self): - self.stubs.Set(os.path, 'exists', lambda p: False) - - self.assertRaises(RuntimeError, - config.find_config_file, - 'glance-foo', {}, []) - - -class TestPasteConfig(unittest.TestCase): - - def test_load_paste_config(self): - path = os.path.join(os.getcwd(), 'etc/glance-api.conf') - - conf = config.load_paste_config(path, 'glance-api') - - self.assertEquals('file', conf['default_store']) - - class TestPasteApp(unittest.TestCase): def setUp(self): @@ -154,15 +36,16 @@ class TestPasteApp(unittest.TestCase): self.stubs.UnsetAll() def test_load_paste_app(self): - path = os.path.join(os.getcwd(), 'etc/glance-api.conf') + conf = config.GlanceConfigOpts() + conf(['--config-file', + os.path.join(os.getcwd(), 'etc/glance-api.conf')]) self.stubs.Set(config, 'setup_logging', lambda *a: None) self.stubs.Set(images, 'create_resource', lambda *a: None) self.stubs.Set(members, 'create_resource', lambda *a: None) - conf, app = config.load_paste_app('glance-api', {}, [path]) + app = config.load_paste_app(conf, 'glance-api') - self.assertEquals('file', conf['default_store']) self.assertEquals(version_negotiation.VersionNegotiationFilter, type(app)) @@ -178,11 +61,12 @@ class TestPasteApp(unittest.TestCase): orig_join = os.path.join self.stubs.Set(os.path, 'join', fake_join) + conf = config.GlanceCacheConfigOpts() + conf([]) + self.stubs.Set(config, 'setup_logging', lambda *a: None) - self.stubs.Set(wsgi, 'app_factory', lambda *a, **kw: 'pruner') + self.stubs.Set(pruner, 'Pruner', lambda conf, **lc: 'pruner') - conf, app = config.load_paste_app('glance-pruner', {}, [], - 'glance-cache') + app = config.load_paste_app(conf, 'glance-pruner') - self.assertEquals('86400', conf['image_cache_stall_time']) self.assertEquals('pruner', app) diff --git a/glance/tests/unit/test_filesystem_store.py b/glance/tests/unit/test_filesystem_store.py index 24c930b7ab..90634f8820 100644 --- a/glance/tests/unit/test_filesystem_store.py +++ b/glance/tests/unit/test_filesystem_store.py @@ -28,6 +28,7 @@ from glance.common import utils from glance.store.location import get_location_from_uri from glance.store.filesystem import Store, ChunkedFile from glance.tests import stubs +from glance.tests import utils as test_utils FILESYSTEM_CONF = { 'verbose': True, @@ -43,7 +44,7 @@ class TestStore(unittest.TestCase): stubs.stub_out_filesystem_backend() self.orig_chunksize = ChunkedFile.CHUNKSIZE ChunkedFile.CHUNKSIZE = 10 - self.store = Store(FILESYSTEM_CONF) + self.store = Store(test_utils.TestConfigOpts(FILESYSTEM_CONF)) def tearDown(self): """Clear the test environment""" diff --git a/glance/tests/unit/test_http_store.py b/glance/tests/unit/test_http_store.py index 93a5355432..6d6eee7015 100644 --- a/glance/tests/unit/test_http_store.py +++ b/glance/tests/unit/test_http_store.py @@ -24,6 +24,7 @@ from glance.common import exception from glance.store import create_stores, delete_from_backend from glance.store.http import Store from glance.store.location import get_location_from_uri +from glance.tests import utils def stub_out_http_backend(stubs): @@ -104,6 +105,6 @@ class TestHttpStore(unittest.TestCase): loc = get_location_from_uri(uri) self.assertRaises(NotImplementedError, self.store.delete, loc) - create_stores({}) + create_stores(utils.TestConfigOpts({})) self.assertRaises(exception.StoreDeleteNotSupported, delete_from_backend, uri) diff --git a/glance/tests/unit/test_image_cache.py b/glance/tests/unit/test_image_cache.py index ad0c870f88..73cf09f944 100644 --- a/glance/tests/unit/test_image_cache.py +++ b/glance/tests/unit/test_image_cache.py @@ -26,6 +26,7 @@ import stubout from glance import image_cache from glance.common import exception from glance.common import utils +from glance.tests import utils as test_utils from glance.tests.utils import skip_if_disabled, xattr_writes_supported FIXTURE_DATA = '*' * 1024 @@ -136,8 +137,7 @@ class ImageCacheTestCase(object): self.assertTrue(os.path.exists(incomplete_file_path)) - self.cache.conf['image_cache_stall_time'] = 0 - self.cache.clean() + self.cache.clean(stall_time=0) self.assertFalse(os.path.exists(incomplete_file_path)) @@ -250,11 +250,12 @@ class TestImageCacheXattr(unittest.TestCase, self.inited = True self.disabled = False - self.conf = {'image_cache_dir': self.cache_dir, - 'image_cache_driver': 'xattr', - 'image_cache_max_size': 1024 * 5, - 'registry_host': '0.0.0.0', - 'registry_port': 9191} + self.conf = test_utils.TestConfigOpts({ + 'image_cache_dir': self.cache_dir, + 'image_cache_driver': 'xattr', + 'image_cache_max_size': 1024 * 5, + 'registry_host': '0.0.0.0', + 'registry_port': 9191}) self.cache = image_cache.ImageCache(self.conf) if not xattr_writes_supported(self.cache_dir): @@ -294,11 +295,12 @@ class TestImageCacheSqlite(unittest.TestCase, self.disabled = False self.cache_dir = os.path.join("/", "tmp", "test.cache.%d" % random.randint(0, 1000000)) - self.conf = {'image_cache_dir': self.cache_dir, - 'image_cache_driver': 'sqlite', - 'image_cache_max_size': 1024 * 5, - 'registry_host': '0.0.0.0', - 'registry_port': 9191} + self.conf = test_utils.TestConfigOpts({ + 'image_cache_dir': self.cache_dir, + 'image_cache_driver': 'sqlite', + 'image_cache_max_size': 1024 * 5, + 'registry_host': '0.0.0.0', + 'registry_port': 9191}) self.cache = image_cache.ImageCache(self.conf) def tearDown(self): diff --git a/glance/tests/unit/test_migrations.py b/glance/tests/unit/test_migrations.py index 2402cb6e94..4708e624aa 100644 --- a/glance/tests/unit/test_migrations.py +++ b/glance/tests/unit/test_migrations.py @@ -34,9 +34,10 @@ from migrate.versioning.repository import Repository from sqlalchemy import * from sqlalchemy.pool import NullPool +from glance.common import cfg from glance.common import exception import glance.registry.db.migration as migration_api -from glance.tests.utils import execute +from glance.tests import utils class TestMigrations(unittest.TestCase): @@ -115,7 +116,7 @@ class TestMigrations(unittest.TestCase): "create database %(database)s;") % locals() cmd = ("mysql -u%(user)s %(password)s -h%(host)s " "-e\"%(sql)s\"") % locals() - exitcode, out, err = execute(cmd) + exitcode, out, err = utils.execute(cmd) self.assertEqual(0, exitcode) def test_walk_versions(self): @@ -124,7 +125,9 @@ class TestMigrations(unittest.TestCase): that there are no errors in the version scripts for each engine """ for key, engine in self.engines.items(): - conf = {'sql_connection': TestMigrations.TEST_DATABASES[key]} + conf = utils.TestConfigOpts({ + 'sql_connection': TestMigrations.TEST_DATABASES[key]}) + conf.register_opt(cfg.StrOpt('sql_connection')) self._walk_versions(conf) def _walk_versions(self, conf): @@ -165,7 +168,9 @@ class TestMigrations(unittest.TestCase): the image_properties table back into the base image table. """ for key, engine in self.engines.items(): - conf = {'sql_connection': TestMigrations.TEST_DATABASES[key]} + conf = utils.TestConfigOpts({ + 'sql_connection': TestMigrations.TEST_DATABASES[key]}) + conf.register_opt(cfg.StrOpt('sql_connection')) self._no_data_loss_2_to_3_to_2(engine, conf) def _no_data_loss_2_to_3_to_2(self, engine, conf): diff --git a/glance/tests/unit/test_misc.py b/glance/tests/unit/test_misc.py index d5039f51bf..f94d0b6ad7 100644 --- a/glance/tests/unit/test_misc.py +++ b/glance/tests/unit/test_misc.py @@ -111,9 +111,9 @@ class UtilsTestCase(unittest.TestCase): # Try importing an object by supplying a class and # verify the object's class name is the same as that supplied - store_obj = utils.import_object('glance.store.s3.Store') + ex_obj = utils.import_object('glance.common.exception.GlanceException') - self.assertTrue(store_obj.__class__.__name__ == 'Store') + self.assertTrue(ex_obj.__class__.__name__ == 'GlanceException') # Try importing a module itself module_obj = utils.import_object('glance.registry') diff --git a/glance/tests/unit/test_notifier.py b/glance/tests/unit/test_notifier.py index fc0c305c85..a26114568e 100644 --- a/glance/tests/unit/test_notifier.py +++ b/glance/tests/unit/test_notifier.py @@ -20,13 +20,14 @@ import unittest from glance.common import exception from glance.common import notifier +from glance.tests import utils class TestInvalidNotifier(unittest.TestCase): """Test that notifications are generated appropriately""" def test_cannot_create(self): - conf = {"notifier_strategy": "invalid_notifier"} + conf = utils.TestConfigOpts({"notifier_strategy": "invalid_notifier"}) self.assertRaises(exception.InvalidNotifierStrategy, notifier.Notifier, conf) @@ -36,7 +37,7 @@ class TestLoggingNotifier(unittest.TestCase): """Test the logging notifier is selected and works properly.""" def setUp(self): - conf = {"notifier_strategy": "logging"} + conf = utils.TestConfigOpts({"notifier_strategy": "logging"}) self.called = False self.logger = logging.getLogger("glance.notifier.logging_notifier") self.notifier = notifier.Notifier(conf) @@ -67,7 +68,7 @@ class TestNoopNotifier(unittest.TestCase): """Test that the noop notifier works...and does nothing?""" def setUp(self): - conf = {"notifier_strategy": "noop"} + conf = utils.TestConfigOpts({"notifier_strategy": "noop"}) self.notifier = notifier.Notifier(conf) def test_warn(self): @@ -86,7 +87,7 @@ class TestRabbitNotifier(unittest.TestCase): def setUp(self): notifier.RabbitStrategy._send_message = self._send_message self.called = False - conf = {"notifier_strategy": "rabbit"} + conf = utils.TestConfigOpts({"notifier_strategy": "rabbit"}) self.notifier = notifier.Notifier(conf) def _send_message(self, message, priority): diff --git a/glance/tests/unit/test_s3_store.py b/glance/tests/unit/test_s3_store.py index bf9d171fb3..e68dfee402 100644 --- a/glance/tests/unit/test_s3_store.py +++ b/glance/tests/unit/test_s3_store.py @@ -32,6 +32,7 @@ from glance.common import utils from glance.store import BackendException, UnsupportedBackend from glance.store.location import get_location_from_uri from glance.store.s3 import Store +from glance.tests import utils as test_utils FAKE_UUID = utils.generate_uuid() @@ -163,7 +164,7 @@ class TestStore(unittest.TestCase): """Establish a clean test environment""" self.stubs = stubout.StubOutForTesting() stub_out_s3(self.stubs) - self.store = Store(S3_CONF) + self.store = Store(test_utils.TestConfigOpts(S3_CONF)) def tearDown(self): """Clear the test environment""" @@ -260,7 +261,7 @@ class TestStore(unittest.TestCase): expected_image_id) image_s3 = StringIO.StringIO(expected_s3_contents) - self.store = Store(new_conf) + self.store = Store(test_utils.TestConfigOpts(new_conf)) location, size, checksum = self.store.add(expected_image_id, image_s3, expected_s3_size) @@ -292,7 +293,7 @@ class TestStore(unittest.TestCase): del conf[key] try: - self.store = Store(conf) + self.store = Store(test_utils.TestConfigOpts(conf)) return self.store.add == self.store.add_disabled except: return False diff --git a/glance/tests/unit/test_store_location.py b/glance/tests/unit/test_store_location.py index 50cd553813..312b93601e 100644 --- a/glance/tests/unit/test_store_location.py +++ b/glance/tests/unit/test_store_location.py @@ -24,8 +24,9 @@ import glance.store.http import glance.store.filesystem import glance.store.swift import glance.store.s3 +from glance.tests import utils -glance.store.create_stores({}) +glance.store.create_stores(utils.TestConfigOpts({})) class TestStoreLocation(unittest.TestCase): diff --git a/glance/tests/unit/test_swift_store.py b/glance/tests/unit/test_swift_store.py index fc2a6771c3..6144a2900b 100644 --- a/glance/tests/unit/test_swift_store.py +++ b/glance/tests/unit/test_swift_store.py @@ -30,6 +30,7 @@ from glance.common import utils from glance.store import BackendException import glance.store.swift from glance.store.location import get_location_from_uri +from glance.tests import utils as test_utils FAKE_UUID = utils.generate_uuid @@ -182,7 +183,7 @@ class TestStore(unittest.TestCase): """Establish a clean test environment""" self.stubs = stubout.StubOutForTesting() stub_out_swift_common_client(self.stubs) - self.store = Store(SWIFT_CONF) + self.store = Store(test_utils.TestConfigOpts(SWIFT_CONF)) def tearDown(self): """Clear the test environment""" @@ -293,7 +294,7 @@ class TestStore(unittest.TestCase): image_swift = StringIO.StringIO(expected_swift_contents) - self.store = Store(new_conf) + self.store = Store(test_utils.TestConfigOpts(new_conf)) location, size, checksum = self.store.add(image_id, image_swift, expected_swift_size) @@ -318,7 +319,7 @@ class TestStore(unittest.TestCase): conf['swift_store_create_container_on_put'] = 'False' conf['swift_store_container'] = 'noexist' image_swift = StringIO.StringIO("nevergonnamakeit") - self.store = Store(conf) + self.store = Store(test_utils.TestConfigOpts(conf)) # We check the exception text to ensure the container # missing text is found in it, otherwise, we would have @@ -348,7 +349,7 @@ class TestStore(unittest.TestCase): '/noexist/%s' % expected_image_id image_swift = StringIO.StringIO(expected_swift_contents) - self.store = Store(conf) + self.store = Store(test_utils.TestConfigOpts(conf)) location, size, checksum = self.store.add(expected_image_id, image_swift, expected_swift_size) @@ -387,7 +388,7 @@ class TestStore(unittest.TestCase): try: glance.store.swift.DEFAULT_LARGE_OBJECT_SIZE = 1024 glance.store.swift.DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 1024 - self.store = Store(conf) + self.store = Store(test_utils.TestConfigOpts(conf)) location, size, checksum = self.store.add(expected_image_id, image_swift, expected_swift_size) @@ -440,7 +441,7 @@ class TestStore(unittest.TestCase): MAX_SWIFT_OBJECT_SIZE = 1024 glance.store.swift.DEFAULT_LARGE_OBJECT_SIZE = 1024 glance.store.swift.DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 1024 - self.store = Store(conf) + self.store = Store(test_utils.TestConfigOpts(conf)) location, size, checksum = self.store.add(expected_image_id, image_swift, 0) finally: @@ -475,7 +476,7 @@ class TestStore(unittest.TestCase): del conf[key] try: - self.store = Store(conf) + self.store = Store(test_utils.TestConfigOpts(conf)) return self.store.add == self.store.add_disabled except: return False diff --git a/glance/tests/unit/test_versions.py b/glance/tests/unit/test_versions.py index 9032e4e322..5341b37068 100644 --- a/glance/tests/unit/test_versions.py +++ b/glance/tests/unit/test_versions.py @@ -22,9 +22,11 @@ import stubout import webob from glance import client +from glance.common import config from glance.common import exception from glance.api import versions from glance.tests import stubs +from glance.tests import utils class VersionsTest(unittest.TestCase): @@ -46,8 +48,10 @@ class VersionsTest(unittest.TestCase): def test_get_version_list(self): req = webob.Request.blank('/') req.accept = "application/json" - conf = {'bind_host': '0.0.0.0', - 'bind_port': 9292} + conf = utils.TestConfigOpts({ + 'bind_host': '0.0.0.0', + 'bind_port': 9292 + }) res = req.get_response(versions.Controller(conf)) self.assertEqual(res.status_int, 300) self.assertEqual(res.content_type, "application/json") diff --git a/glance/tests/utils.py b/glance/tests/utils.py index 97e21d0c02..3f735ca13c 100644 --- a/glance/tests/utils.py +++ b/glance/tests/utils.py @@ -22,9 +22,44 @@ import functools import os import socket import subprocess +import tempfile import nose.plugins.skip +from glance.common import config + + +class TestConfigOpts(config.GlanceConfigOpts): + + def __init__(self, test_values): + super(TestConfigOpts, self).__init__() + self._test_values = test_values + self() + + def __call__(self): + config_file = self._write_tmp_config_file() + try: + super(TestConfigOpts, self).\ + __call__(['--config-file', config_file]) + finally: + os.remove(config_file) + + def _write_tmp_config_file(self): + contents = '[DEFAULT]\n' + for key, value in self._test_values.items(): + contents += '%s = %s\n' % (key, value) + + (fd, path) = tempfile.mkstemp(prefix='testcfg') + try: + os.write(fd, contents) + except Exception, e: + os.close(fd) + os.remove(path) + raise e + + os.close(fd) + return path + class skip_test(object): """Decorator that skips a test."""