From c9b0f65f1892abc2732fcf218546cfae366f846f Mon Sep 17 00:00:00 2001 From: Flavio Percoco Date: Thu, 3 Oct 2013 14:57:32 +0200 Subject: [PATCH] Use oslo.config directly instead of common.config We've been using a wrapper around oslo.config that worked well at the early stages of the project. Now, as the project grows, we need to user more features from oslo.config that wouldn't make sense to export through the wrapper we have. This patch deprecates marconi.common.config and uses oslo.config directly where needed. This is the first patch around the bp remove-global-config, upcoming patches will completely remove the usage of a global instance. Partially-implements remove-global-config Change-Id: Ie14c6bae8cdb72a6ed93a12c34cd8b156e059872 --- marconi/common/cli.py | 3 - marconi/common/config.py | 200 ------------------ marconi/proxy/base.py | 28 ++- marconi/proxy/storage/mongodb/options.py | 20 +- marconi/proxy/transport/__init__.py | 7 +- marconi/proxy/transport/wsgi/driver.py | 26 ++- marconi/queues/bootstrap.py | 26 ++- marconi/queues/storage/__init__.py | 11 + marconi/queues/storage/mongodb/claims.py | 8 +- marconi/queues/storage/mongodb/messages.py | 8 +- marconi/queues/storage/mongodb/options.py | 59 +++--- marconi/queues/storage/mongodb/queues.py | 8 +- marconi/queues/storage/sqlite/claims.py | 9 +- marconi/queues/storage/sqlite/driver.py | 11 +- marconi/queues/storage/sqlite/messages.py | 9 +- marconi/queues/storage/sqlite/queues.py | 9 +- marconi/queues/transport/__init__.py | 11 +- marconi/queues/transport/validation.py | 24 ++- marconi/queues/transport/wsgi/__init__.py | 6 +- marconi/queues/transport/wsgi/claims.py | 6 +- marconi/queues/transport/wsgi/configs.py | 29 +++ marconi/queues/transport/wsgi/driver.py | 16 +- marconi/queues/transport/wsgi/messages.py | 6 +- marconi/queues/transport/wsgi/metadata.py | 6 +- marconi/tests/base.py | 8 +- marconi/tests/functional/base.py | 2 +- tests/unit/queues/transport/test_auth.py | 9 +- .../unit/queues/transport/wsgi/test_claims.py | 9 +- .../queues/transport/wsgi/test_messages.py | 5 +- .../transport/wsgi/test_queue_lifecycle.py | 9 +- 30 files changed, 214 insertions(+), 374 deletions(-) delete mode 100644 marconi/common/config.py create mode 100644 marconi/queues/transport/wsgi/configs.py diff --git a/marconi/common/cli.py b/marconi/common/cli.py index ff2df983f..6ef085d4a 100644 --- a/marconi/common/cli.py +++ b/marconi/common/cli.py @@ -20,10 +20,8 @@ import functools import sys import termios -from marconi.common import config from marconi.openstack.common import log as logging -PROJECT_CFG = config.project('marconi') LOG = logging.getLogger(__name__) @@ -69,7 +67,6 @@ def runnable(func): try: logging.setup('marconi') - PROJECT_CFG.load(args=sys.argv[1:]) func() except KeyboardInterrupt: LOG.info(_(u'Terminating')) diff --git a/marconi/common/config.py b/marconi/common/config.py deleted file mode 100644 index a30654313..000000000 --- a/marconi/common/config.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright (c) 2013 Rackspace, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Decentralized configuration module. - -A config variable `foo` is a read-only property accessible through - - CFG.foo - -, where `CFG` is either a global configuration accessible through - - CFG = config.project('marconi').from_options( - foo=('bar', 'usage'), - ...) - -, or a local configuration associated with a namespace - - CFG = config.namespace('drivers:transport:wsgi').from_options( - port=80, - ...) - -The `from_options` call accepts a list of option definition, where each -option is represented as a keyword argument, in the form of either -`name=default` or `name=(default, description)`, where `name` is the -name of the option in a valid Python identifier, and `default` is the -default value of that option. - -Configurations can be read from an INI file, where the global options -are under the `[DEFAULT]` section, and the local options are under the -sections named by their associated namespaces. - -To load the configurations from a file: - - PROJECT_CFG = config.project('marconi') - PROJECT_CFG.load(filename='/path/to/example.conf') - -A call to `.load` without a filename looks up for the default ones: - - ~/.marconi/marconi.conf - /etc/marconi/marconi.conf - -Global config variables, if any, can also be read from the command line -arguments: - - PROJECT_CFG.load(filename='example.conf', args=sys.argv[1:]) -""" - -from oslo.config import cfg - - -def _init(): - """Enclose an API specific config object.""" - - class ConfigProxy(object): - """Prototype of the opaque config variable accessors.""" - pass - - class Obj(dict): - __getattr__ = dict.__getitem__ - __setattr__ = dict.__setitem__ - - conf = cfg.CONF - - def namespace(name, title=None): - """Create a config namespace. - - :param name: the section name appears in the .ini file - :param title: an optional description - :returns: the option object for the namespace - """ - - grp = cfg.OptGroup(name, title) - conf.register_group(grp) - - def from_options(**opts): - """Define options under the associated namespace. - - :returns: ConfigProxy of the associated namespace - """ - - for k, v in opts.items(): - conf.register_opt(_make_opt(k, v), group=grp) - - def from_class(cls): - grant_access_to_class(conf[grp.name], cls) - return cls - - return from_class(opaque_type_of(ConfigProxy, grp.name))() - - return Obj(from_options=from_options) - - def project(name=None, prog=None): - """Access the global namespace. - - :param name: the name of the project - :returns: a global option object - """ - - def from_options(**opts): - """Define options under the global namespace. - - :returns: ConfigProxy of the global namespace - """ - - for k, v in opts.items(): - conf.register_cli_opt(_make_opt(k, v)) - - def from_class(cls): - grant_access_to_class(conf, cls) - return cls - - return from_class(opaque_type_of(ConfigProxy, name))() - - def load(filename=None, args=None): - """Load the configurations from a config file. - - If the file name is not supplied, look for - - ~/.%project/%project.conf - - and - - /etc/%project/%project.conf - - :param filename: the name of an alternative config file - :param args: command line arguments - """ - - args = [] if args is None else args - - if filename is None: - conf(args=args, project=name, prog=prog) - else: - conf(args=args, default_config_files=[filename]) - - return Obj(from_options=from_options, load=load, conf=conf) - - def opaque_type_of(base, postfix): - return type('%s of %s' % (base.__name__, postfix), (base,), {}) - - def grant_access_to_class(pairs, cls): - for k in pairs: - # A closure is needed for each %k to let - # different properties access different %k. - def let(k=k): - setattr(cls, k, property(lambda obj: pairs[k])) - let() - - return namespace, project - - -namespace, project = _init() - - -def _make_opt(name, default): - """Create an oslo.config option with type deduction - - The type for the option is deduced from the %default value given - for that option. A default value of None is deduced to Opt. - - Note: MultiStrOpt is not supported. - - :param name: the name of the option in a valid Python identifier - :param default: the default value of the option, or (default, description) - :raises: cfg.Error if the type can not be deduced. - """ - - deduction = { - str: cfg.StrOpt, - bool: cfg.BoolOpt, - int: cfg.IntOpt, - long: cfg.IntOpt, - float: cfg.FloatOpt, - list: cfg.ListOpt, - } - - if type(default) is tuple: - default, help = default - else: - help = None - - if default is None: - return cfg.Opt(name, help=help) - - try: - return deduction[type(default)](name, help=help, default=default) - except KeyError: - raise cfg.Error(u'unrecognized option type') diff --git a/marconi/proxy/base.py b/marconi/proxy/base.py index e5ca81d72..184175ad9 100644 --- a/marconi/proxy/base.py +++ b/marconi/proxy/base.py @@ -18,17 +18,21 @@ from stevedore import driver from marconi.common import access from marconi.common.cache import cache as oslo_cache -from marconi.common import config from marconi.common import decorators from marconi.common import exceptions from marconi.openstack.common import log from marconi.proxy import transport # NOQA -PROJECT_CFG = config.project('marconi', 'marconi-proxy') -CFG = config.namespace('proxy:drivers').from_options( - transport='wsgi', - storage='memory') +_bootstrap_options = [ + cfg.StrOpt('transport', default='wsgi', + help='Transport driver to use'), + cfg.StrOpt('storage', default='memory', + help='Storage driver to use'), +] + +CFG = cfg.CONF +CFG.register_opts(_bootstrap_options, group="proxy:drivers") LOG = log.getLogger(__name__) @@ -41,7 +45,13 @@ class Bootstrap(object): """ def __init__(self, access_mode, config_file=None, cli_args=None): - PROJECT_CFG.load(filename=config_file, args=cli_args) + default_file = None + if config_file is not None: + default_file = [config_file] + + CFG(project='marconi', args=cli_args or [], + default_config_files=default_file) + log.setup('marconi_proxy') form = 'marconi.proxy.{0}.transport' lookup = {access.Access.public: 'public', @@ -53,7 +63,7 @@ class Bootstrap(object): LOG.debug(_(u'Loading Proxy Storage Driver')) try: mgr = driver.DriverManager('marconi.proxy.storage', - CFG.storage, + CFG['proxy:drivers'].storage, invoke_on_load=True) return mgr.driver except RuntimeError as exc: @@ -64,7 +74,7 @@ class Bootstrap(object): def cache(self): LOG.debug(_(u'Loading Proxy Cache Driver')) try: - mgr = oslo_cache.get_cache(cfg.CONF) + mgr = oslo_cache.get_cache(CFG) return mgr except RuntimeError as exc: LOG.exception(exc) @@ -75,7 +85,7 @@ class Bootstrap(object): LOG.debug(_(u'Loading Proxy Transport Driver')) try: mgr = driver.DriverManager(self._transport_type, - CFG.transport, + CFG['proxy:drivers'].transport, invoke_on_load=True, invoke_args=[self.storage, self.cache]) diff --git a/marconi/proxy/storage/mongodb/options.py b/marconi/proxy/storage/mongodb/options.py index b79b5f963..45234eda4 100644 --- a/marconi/proxy/storage/mongodb/options.py +++ b/marconi/proxy/storage/mongodb/options.py @@ -15,16 +15,18 @@ """MongoDB driver configuration options.""" -from marconi.common import config +from oslo.config import cfg -OPTIONS = { - # Connection string - 'uri': None, + +_MONGODB_OPTIONS = [ + cfg.StrOpt('uri', help='Mongodb Connection URI'), # Database name - 'database': 'marconi_proxy' -} + # TODO(kgriffs): Consider local sharding across DBs to mitigate + # per-DB locking latency. + cfg.StrOpt('database', default='marconi_proxy', help='Database name'), +] -CFG = config.namespace('proxy:drivers:storage:mongodb').from_options( - **OPTIONS -) +_GROUP = 'proxy:drivers:storage:mongodb' +cfg.CONF.register_opts(_MONGODB_OPTIONS, group=_GROUP) +CFG = cfg.CONF[_GROUP] diff --git a/marconi/proxy/transport/__init__.py b/marconi/proxy/transport/__init__.py index 188146bf9..76cf7e849 100644 --- a/marconi/proxy/transport/__init__.py +++ b/marconi/proxy/transport/__init__.py @@ -1,9 +1,12 @@ """Marconi Proxy Transport Drivers""" -from marconi.common import config +from oslo.config import cfg + from marconi.proxy.transport import base -CFG = config.project('marconi').from_options() +# NOTE(flaper87): Not sure +# what this is for. +CFG = cfg.CONF # Hoist into package namespace DriverBase = base.DriverBase diff --git a/marconi/proxy/transport/wsgi/driver.py b/marconi/proxy/transport/wsgi/driver.py index e7c0548cf..13b99e457 100644 --- a/marconi/proxy/transport/wsgi/driver.py +++ b/marconi/proxy/transport/wsgi/driver.py @@ -17,9 +17,9 @@ import abc from wsgiref import simple_server import falcon +from oslo.config import cfg import six -from marconi.common import config from marconi.common.transport.wsgi import helpers import marconi.openstack.common.log as logging from marconi.proxy import transport @@ -28,16 +28,20 @@ from marconi.proxy.utils import round_robin from marconi.queues.transport import auth -OPTIONS = { - 'bind': '0.0.0.0', - 'port': 8889 -} +_WSGI_OPTIONS = [ + cfg.StrOpt('bind', default='0.0.0.0', + help='Address to bind this server to'), -PROJECT_CFG = config.project('marconi', 'marconi-proxy') -GLOBAL_CFG = PROJECT_CFG.from_options() -WSGI_CFG = config.namespace('proxy:drivers:transport:wsgi').from_options( - **OPTIONS -) + cfg.IntOpt('port', default=8888, + help='Port to bind this server to'), + +] + +cfg.CONF.register_opts(_WSGI_OPTIONS, + group='proxy:drivers:transport:wsgi') + +GLOBAL_CFG = cfg.CONF +WSGI_CFG = cfg.CONF['proxy:drivers:transport:wsgi'] LOG = logging.getLogger(__name__) @@ -74,7 +78,7 @@ class DriverBase(transport.DriverBase): # NOTE(flaper87): Install Auth if GLOBAL_CFG.auth_strategy: strategy = auth.strategy(GLOBAL_CFG.auth_strategy) - self.app = strategy.install(self.app, PROJECT_CFG.conf) + self.app = strategy.install(self.app, GLOBAL_CFG) @abc.abstractproperty def bridge(self): diff --git a/marconi/queues/bootstrap.py b/marconi/queues/bootstrap.py index cd88ae033..7844e86ae 100644 --- a/marconi/queues/bootstrap.py +++ b/marconi/queues/bootstrap.py @@ -13,19 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +from oslo.config import cfg from stevedore import driver -from marconi.common import config from marconi.common import decorators from marconi.common import exceptions from marconi.openstack.common import log from marconi.queues import transport # NOQA -PROJECT_CFG = config.project('marconi', 'marconi-queues') -CFG = config.namespace('queues:drivers').from_options( - transport='wsgi', - storage='sqlite') +_bootstrap_options = [ + cfg.StrOpt('transport', default='wsgi', + help='Transport driver to use'), + cfg.StrOpt('storage', default='sqlite', + help='Storage driver to use'), +] + +CFG = cfg.CONF +CFG.register_opts(_bootstrap_options, group="queues:drivers") LOG = log.getLogger(__name__) @@ -38,7 +43,12 @@ class Bootstrap(object): """ def __init__(self, config_file=None, cli_args=None): - PROJECT_CFG.load(filename=config_file, args=cli_args) + default_file = None + if config_file is not None: + default_file = [config_file] + + CFG(project='marconi', prog='marconi-queues', args=cli_args or [], + default_config_files=default_file) log.setup('marconi') @decorators.lazy_property(write=False) @@ -46,7 +56,7 @@ class Bootstrap(object): LOG.debug(_(u'Loading Storage Driver')) try: mgr = driver.DriverManager('marconi.queues.storage', - CFG.storage, + CFG['queues:drivers'].storage, invoke_on_load=True) return mgr.driver except RuntimeError as exc: @@ -58,7 +68,7 @@ class Bootstrap(object): LOG.debug(_(u'Loading Transport Driver')) try: mgr = driver.DriverManager('marconi.queues.transport', - CFG.transport, + CFG['queues:drivers'].transport, invoke_on_load=True, invoke_args=[self.storage]) return mgr.driver diff --git a/marconi/queues/storage/__init__.py b/marconi/queues/storage/__init__.py index 17aa6830e..2f6a5c907 100644 --- a/marconi/queues/storage/__init__.py +++ b/marconi/queues/storage/__init__.py @@ -1,8 +1,19 @@ """Marconi Storage Drivers""" +from oslo.config import cfg + from marconi.queues.storage import base from marconi.queues.storage import exceptions # NOQA +_STORAGE_LIMITS_OPTIONS = [ + cfg.IntOpt('default_queue_paging', default=10, + help='Default queue pagination size'), + + cfg.IntOpt('default_message_paging', default=10, + help='Default message pagination size') +] + +cfg.CONF.register_opts(_STORAGE_LIMITS_OPTIONS, group='queues:limits:storage') # Hoist classes into package namespace ClaimBase = base.ClaimBase diff --git a/marconi/queues/storage/mongodb/claims.py b/marconi/queues/storage/mongodb/claims.py index f00bd358f..01015dfd8 100644 --- a/marconi/queues/storage/mongodb/claims.py +++ b/marconi/queues/storage/mongodb/claims.py @@ -24,8 +24,8 @@ Field Mappings: import datetime from bson import objectid +from oslo.config import cfg -from marconi.common import config import marconi.openstack.common.log as logging from marconi.openstack.common import timeutils from marconi.queues import storage @@ -33,9 +33,7 @@ from marconi.queues.storage import exceptions from marconi.queues.storage.mongodb import utils LOG = logging.getLogger(__name__) -CFG = config.namespace('queues:limits:storage').from_options( - default_message_paging=10, -) +STORAGE_LIMITS = cfg.CONF['queues:limits:storage'] class ClaimController(storage.ClaimBase): @@ -122,7 +120,7 @@ class ClaimController(storage.ClaimBase): msg_ctrl = self.driver.message_controller if limit is None: - limit = CFG.default_message_paging + limit = STORAGE_LIMITS.default_message_paging ttl = metadata['ttl'] grace = metadata['grace'] diff --git a/marconi/queues/storage/mongodb/messages.py b/marconi/queues/storage/mongodb/messages.py index be6481af7..b5dd3ad70 100644 --- a/marconi/queues/storage/mongodb/messages.py +++ b/marconi/queues/storage/mongodb/messages.py @@ -24,10 +24,10 @@ Field Mappings: import datetime import time +from oslo.config import cfg import pymongo.errors import pymongo.read_preferences -from marconi.common import config import marconi.openstack.common.log as logging from marconi.openstack.common import timeutils from marconi.queues import storage @@ -36,9 +36,7 @@ from marconi.queues.storage.mongodb import options from marconi.queues.storage.mongodb import utils LOG = logging.getLogger(__name__) -CFG = config.namespace('queues:limits:storage').from_options( - default_message_paging=10, -) +STORAGE_LIMITS = cfg.CONF['queues:limits:storage'] # NOTE(kgriffs): This value, in seconds, should be at least less than the # minimum allowed TTL for messages (60 seconds). Make it 45 to allow for @@ -354,7 +352,7 @@ class MessageController(storage.MessageBase): echo=False, client_uuid=None, include_claimed=False): if limit is None: - limit = CFG.default_message_paging + limit = STORAGE_LIMITS.default_message_paging if marker is not None: try: diff --git a/marconi/queues/storage/mongodb/options.py b/marconi/queues/storage/mongodb/options.py index c2dde575e..970976ac7 100644 --- a/marconi/queues/storage/mongodb/options.py +++ b/marconi/queues/storage/mongodb/options.py @@ -16,38 +16,43 @@ """MongoDB storage driver configuration options.""" -from marconi.common import config +from oslo.config import cfg -# TODO(kgriffs): Expose documentation to oslo.config -OPTIONS = { - # Connection string - 'uri': None, +_MONGODB_OPTIONS = [ + cfg.StrOpt('uri', help='Mongodb Connection URI'), - # Database name. Will be postfixed with partition identifiers. - 'database': 'marconi', + # Database name + # TODO(kgriffs): Consider local sharding across DBs to mitigate + # per-DB locking latency. + cfg.StrOpt('database', default='marconi', help='Database name'), - # Number of databases across which to partition message data, - # in order to reduce writer lock %. DO NOT change this setting - # after initial deployment. It MUST remain static. Also, - # you should not need a large number of partitions to improve - # performance, esp. if deploying MongoDB on SSD storage. - 'partitions': 2, + cfg.IntOpt('partitions', default=2, + help=('Number of databases across which to ' + 'partition message data, in order to ' + 'reduce writer lock %. DO NOT change ' + 'this setting after initial deployment. ' + 'It MUST remain static. Also, you ' + 'should not need a large number of partitions ' + 'to improve performance, esp. if deploying ' + 'MongoDB on SSD storage.')), - # Maximum number of times to retry a failed operation. Currently - # only used for retrying a message post. - 'max_attempts': 1000, + cfg.IntOpt('max_attempts', default=1000, + help=('Maximum number of times to retry a failed operation.' + 'Currently only used for retrying a message post.')), - # Maximum sleep interval between retries (actual sleep time - # increases linearly according to number of attempts performed). - 'max_retry_sleep': 0.1, + cfg.FloatOpt('max_retry_sleep', default=0.1, + help=('Maximum sleep interval between retries ' + '(actual sleep time increases linearly ' + 'according to number of attempts performed).')), - # Maximum jitter interval, to be added to the sleep interval, in - # order to decrease probability that parallel requests will retry - # at the same instant. - 'max_retry_jitter': 0.005, -} + cfg.FloatOpt('max_retry_jitter', default=0.005, + help=('Maximum jitter interval, to be added to the ' + 'sleep interval, in order to decrease probability ' + 'that parallel requests will retry at the ' + 'same instant.')), +] -CFG = config.namespace('queues:drivers:storage:mongodb').from_options( - **OPTIONS -) +cfg.CONF.register_opts(_MONGODB_OPTIONS, + group='queues:drivers:storage:mongodb') +CFG = cfg.CONF['queues:drivers:storage:mongodb'] diff --git a/marconi/queues/storage/mongodb/queues.py b/marconi/queues/storage/mongodb/queues.py index eafb34be0..4e515009b 100644 --- a/marconi/queues/storage/mongodb/queues.py +++ b/marconi/queues/storage/mongodb/queues.py @@ -21,9 +21,9 @@ Field Mappings: letter of their long name. """ +from oslo.config import cfg import pymongo.errors -from marconi.common import config import marconi.openstack.common.log as logging from marconi.openstack.common import timeutils from marconi.queues import storage @@ -31,9 +31,7 @@ from marconi.queues.storage import exceptions from marconi.queues.storage.mongodb import utils LOG = logging.getLogger(__name__) -CFG = config.namespace('queues:limits:storage').from_options( - default_queue_paging=10, -) +STORAGE_LIMITS = cfg.CONF['queues:limits:storage'] class QueueController(storage.QueueBase): @@ -174,7 +172,7 @@ class QueueController(storage.QueueBase): limit=None, detailed=False): if limit is None: - limit = CFG.default_queue_paging + limit = STORAGE_LIMITS.default_queue_paging query = {} scoped_name = utils.scope_queue_name(marker, project) diff --git a/marconi/queues/storage/sqlite/claims.py b/marconi/queues/storage/sqlite/claims.py index 5c2b52550..b4b7d166c 100644 --- a/marconi/queues/storage/sqlite/claims.py +++ b/marconi/queues/storage/sqlite/claims.py @@ -13,14 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from marconi.common import config +from oslo.config import cfg + from marconi.queues.storage import base from marconi.queues.storage import exceptions from marconi.queues.storage.sqlite import utils -CFG = config.namespace('queues:limits:storage').from_options( - default_message_paging=10, -) +STORAGE_LIMITS = cfg.CONF['queues:limits:storage'] class ClaimController(base.ClaimBase): @@ -84,7 +83,7 @@ class ClaimController(base.ClaimBase): project = '' if limit is None: - limit = CFG.default_message_paging + limit = STORAGE_LIMITS.default_message_paging with self.driver('immediate'): try: diff --git a/marconi/queues/storage/sqlite/driver.py b/marconi/queues/storage/sqlite/driver.py index 5d0f17545..345988617 100644 --- a/marconi/queues/storage/sqlite/driver.py +++ b/marconi/queues/storage/sqlite/driver.py @@ -18,14 +18,19 @@ import sqlite3 import uuid import msgpack +from oslo.config import cfg -from marconi.common import config from marconi.queues import storage from marconi.queues.storage.sqlite import controllers from marconi.queues.storage.sqlite import utils -CFG = config.namespace('queues:drivers:storage:sqlite').from_options( - database=':memory:') +_SQLITE_OPTIONS = [ + cfg.StrOpt('database', default=':memory:', + help='Sqlite database to use.') +] + +cfg.CONF.register_opts(_SQLITE_OPTIONS, group='queues:drivers:storage:sqlite') +CFG = cfg.CONF['queues:drivers:storage:sqlite'] class Driver(storage.DriverBase): diff --git a/marconi/queues/storage/sqlite/messages.py b/marconi/queues/storage/sqlite/messages.py index caf908a3e..d4c090374 100644 --- a/marconi/queues/storage/sqlite/messages.py +++ b/marconi/queues/storage/sqlite/messages.py @@ -13,15 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from marconi.common import config +from oslo.config import cfg + from marconi.openstack.common import timeutils from marconi.queues.storage import base from marconi.queues.storage import exceptions from marconi.queues.storage.sqlite import utils -CFG = config.namespace('queues:limits:storage').from_options( - default_message_paging=10, -) +STORAGE_LIMITS = cfg.CONF['queues:limits:storage'] class MessageController(base.MessageBase): @@ -139,7 +138,7 @@ class MessageController(base.MessageBase): echo=False, client_uuid=None, include_claimed=False): if limit is None: - limit = CFG.default_message_paging + limit = STORAGE_LIMITS.default_message_paging if project is None: project = '' diff --git a/marconi/queues/storage/sqlite/queues.py b/marconi/queues/storage/sqlite/queues.py index 3353676fd..cfc29476e 100644 --- a/marconi/queues/storage/sqlite/queues.py +++ b/marconi/queues/storage/sqlite/queues.py @@ -14,14 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from marconi.common import config +from oslo.config import cfg + from marconi.queues.storage import base from marconi.queues.storage import exceptions from marconi.queues.storage.sqlite import utils -CFG = config.namespace('queues:limits:storage').from_options( - default_queue_paging=10, -) +STORAGE_LIMITS = cfg.CONF['queues:limits:storage'] class QueueController(base.QueueBase): @@ -47,7 +46,7 @@ class QueueController(base.QueueBase): project = '' if limit is None: - limit = CFG.default_queue_paging + limit = STORAGE_LIMITS.default_queue_paging sql = ((''' select name from Queues''' if not detailed diff --git a/marconi/queues/transport/__init__.py b/marconi/queues/transport/__init__.py index cec61b92b..62747f29c 100644 --- a/marconi/queues/transport/__init__.py +++ b/marconi/queues/transport/__init__.py @@ -1,13 +1,14 @@ """Marconi Transport Drivers""" -from marconi.common import config +from oslo.config import cfg + from marconi.queues.transport import base -OPTIONS = { - 'auth_strategy': "" -} +_TRANSPORT_OPTIONS = [ + cfg.StrOpt('auth_strategy', default='') +] -CFG = config.project('marconi').from_options(**OPTIONS) +cfg.CONF.register_opts(_TRANSPORT_OPTIONS) # Hoist into package namespace DriverBase = base.DriverBase diff --git a/marconi/queues/transport/validation.py b/marconi/queues/transport/validation.py index 131a62013..a7ab1aa85 100644 --- a/marconi/queues/transport/validation.py +++ b/marconi/queues/transport/validation.py @@ -15,21 +15,23 @@ import re +from oslo.config import cfg import simplejson as json -from marconi.common import config -OPTIONS = { - 'queue_paging_uplimit': 20, - 'metadata_size_uplimit': 64 * 1024, - 'message_paging_uplimit': 20, - 'message_size_uplimit': 256 * 1024, - 'message_ttl_max': 1209600, - 'claim_ttl_max': 43200, - 'claim_grace_max': 43200, -} +_TRANSPORT_LIMITS_OPTIONS = [ + cfg.IntOpt('queue_paging_uplimit', default=20), + cfg.IntOpt('metadata_size_uplimit', default=64 * 1024), + cfg.IntOpt('message_paging_uplimit', default=20), + cfg.IntOpt('message_size_uplimit', default=256 * 1024), + cfg.IntOpt('message_ttl_max', default=1209600), + cfg.IntOpt('claim_ttl_max', default=43200), + cfg.IntOpt('claim_grace_max', default=43200), +] -CFG = config.namespace('queues:limits:transport').from_options(**OPTIONS) +cfg.CONF.register_opts(_TRANSPORT_LIMITS_OPTIONS, + group='queues:limits:transport') +CFG = cfg.CONF['queues:limits:transport'] # NOTE(kgriffs): Don't use \w because it isn't guaranteed to match # only ASCII characters. diff --git a/marconi/queues/transport/wsgi/__init__.py b/marconi/queues/transport/wsgi/__init__.py index d268cda88..792f6cd52 100644 --- a/marconi/queues/transport/wsgi/__init__.py +++ b/marconi/queues/transport/wsgi/__init__.py @@ -1,11 +1,7 @@ """WSGI Transport Driver""" -from marconi.queues.transport.wsgi import claims # NOQA +from marconi.queues.transport.wsgi import configs # NOQA from marconi.queues.transport.wsgi import driver -from marconi.queues.transport.wsgi import messages # NOQA -from marconi.queues.transport.wsgi import queues # NOQA -from marconi.queues.transport.wsgi import stats # NOQA - # Hoist into package namespace Driver = driver.Driver diff --git a/marconi/queues/transport/wsgi/claims.py b/marconi/queues/transport/wsgi/claims.py index 333cf0e69..d9aeb8abd 100644 --- a/marconi/queues/transport/wsgi/claims.py +++ b/marconi/queues/transport/wsgi/claims.py @@ -14,9 +14,9 @@ # limitations under the License. import falcon +from oslo.config import cfg import six -from marconi.common import config import marconi.openstack.common.log as logging from marconi.queues.storage import exceptions as storage_exceptions from marconi.queues.transport import utils @@ -26,9 +26,7 @@ from marconi.queues.transport.wsgi import utils as wsgi_utils LOG = logging.getLogger(__name__) -CFG = config.namespace('queues:drivers:transport:wsgi').from_options( - metadata_max_length=64 * 1024 -) +CFG = cfg.CONF['queues:drivers:transport:wsgi'] CLAIM_POST_SPEC = (('ttl', int), ('grace', int)) CLAIM_PATCH_SPEC = (('ttl', int),) diff --git a/marconi/queues/transport/wsgi/configs.py b/marconi/queues/transport/wsgi/configs.py new file mode 100644 index 000000000..cf0fc3ec0 --- /dev/null +++ b/marconi/queues/transport/wsgi/configs.py @@ -0,0 +1,29 @@ +# Copyright (c) 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo.config import cfg + +_WSGI_OPTIONS = [ + cfg.StrOpt('bind', default='0.0.0.0', + help='Address to bind this server to'), + + cfg.IntOpt('port', default=8888, + help='Port to bind this server to'), + + cfg.IntOpt('content_max_length', default=256 * 1024), + cfg.IntOpt('metadata_max_length', default=64 * 1024) +] + +cfg.CONF.register_opts(_WSGI_OPTIONS, group='queues:drivers:transport:wsgi') diff --git a/marconi/queues/transport/wsgi/driver.py b/marconi/queues/transport/wsgi/driver.py index c9b14b485..14ec0c91e 100644 --- a/marconi/queues/transport/wsgi/driver.py +++ b/marconi/queues/transport/wsgi/driver.py @@ -14,9 +14,9 @@ # limitations under the License. import falcon +from oslo.config import cfg from wsgiref import simple_server -from marconi.common import config from marconi.common.transport.wsgi import helpers import marconi.openstack.common.log as logging from marconi.queues import transport @@ -30,16 +30,8 @@ from marconi.queues.transport.wsgi import stats from marconi.queues.transport.wsgi import v1 -OPTIONS = { - 'bind': '0.0.0.0', - 'port': 8888 -} - -PROJECT_CFG = config.project('marconi', 'marconi-queues') -GLOBAL_CFG = PROJECT_CFG.from_options() -WSGI_CFG = config.namespace('queues:drivers:transport:wsgi').from_options( - **OPTIONS -) +GLOBAL_CFG = cfg.CONF +WSGI_CFG = cfg.CONF['queues:drivers:transport:wsgi'] LOG = logging.getLogger(__name__) @@ -114,7 +106,7 @@ class Driver(transport.DriverBase): # NOTE(flaper87): Install Auth if GLOBAL_CFG.auth_strategy: strategy = auth.strategy(GLOBAL_CFG.auth_strategy) - self.app = strategy.install(self.app, PROJECT_CFG.conf) + self.app = strategy.install(self.app, GLOBAL_CFG) def listen(self): """Self-host using 'bind' and 'port' from the WSGI config group.""" diff --git a/marconi/queues/transport/wsgi/messages.py b/marconi/queues/transport/wsgi/messages.py index 89f509fd9..10440894e 100644 --- a/marconi/queues/transport/wsgi/messages.py +++ b/marconi/queues/transport/wsgi/messages.py @@ -14,9 +14,9 @@ # limitations under the License. import falcon +from oslo.config import cfg import six -from marconi.common import config import marconi.openstack.common.log as logging from marconi.queues.storage import exceptions as storage_exceptions from marconi.queues.transport import utils @@ -26,9 +26,7 @@ from marconi.queues.transport.wsgi import utils as wsgi_utils LOG = logging.getLogger(__name__) -CFG = config.namespace('queues:drivers:transport:wsgi').from_options( - content_max_length=256 * 1024 -) +CFG = cfg.CONF['queues:drivers:transport:wsgi'] MESSAGE_POST_SPEC = (('ttl', int), ('body', '*')) diff --git a/marconi/queues/transport/wsgi/metadata.py b/marconi/queues/transport/wsgi/metadata.py index 2fa61ad43..14e541489 100644 --- a/marconi/queues/transport/wsgi/metadata.py +++ b/marconi/queues/transport/wsgi/metadata.py @@ -14,9 +14,9 @@ # limitations under the License. import falcon +from oslo.config import cfg import six -from marconi.common import config import marconi.openstack.common.log as logging from marconi.queues.storage import exceptions as storage_exceptions from marconi.queues.transport import utils @@ -25,9 +25,7 @@ from marconi.queues.transport.wsgi import exceptions as wsgi_exceptions from marconi.queues.transport.wsgi import utils as wsgi_utils LOG = logging.getLogger(__name__) -CFG = config.namespace('queues:drivers:transport:wsgi').from_options( - metadata_max_length=64 * 1024 -) +CFG = cfg.CONF['queues:drivers:transport:wsgi'] class Resource(object): diff --git a/marconi/tests/base.py b/marconi/tests/base.py index 2ce997059..9e20809b6 100644 --- a/marconi/tests/base.py +++ b/marconi/tests/base.py @@ -17,9 +17,7 @@ import fixtures import os import testtools -from marconi.common import config - -CFG = config.project() +from oslo.config import cfg class TestBase(testtools.TestCase): @@ -65,8 +63,8 @@ class TestBase(testtools.TestCase): :returns: Project's config object. """ - CFG.load(filename=cls.conf_path(filename)) - return CFG + cfg.CONF(args=[], default_config_files=[cls.conf_path(filename)]) + return cfg.CONF def config(self, group=None, **kw): """Override some configuration values. diff --git a/marconi/tests/functional/base.py b/marconi/tests/functional/base.py index f84e1a856..9a0cff0d7 100644 --- a/marconi/tests/functional/base.py +++ b/marconi/tests/functional/base.py @@ -51,7 +51,7 @@ class FunctionalTestBase(testing.TestBase): self.server = self.server_class() self.server.start(self.conf_path(self.cfg.marconi.config)) - self.mconf = self.load_conf(self.cfg.marconi.config).conf + self.mconf = self.load_conf(self.cfg.marconi.config) self.limits = self.mconf['queues:limits:transport'] # NOTE(flaper87): Create client diff --git a/tests/unit/queues/transport/test_auth.py b/tests/unit/queues/transport/test_auth.py index e8170df8f..d7bfc8d4b 100644 --- a/tests/unit/queues/transport/test_auth.py +++ b/tests/unit/queues/transport/test_auth.py @@ -16,7 +16,6 @@ from oslo.config import cfg -from marconi.common import config from marconi.queues.transport import auth from marconi import tests as testing @@ -25,13 +24,11 @@ class TestTransportAuth(testing.TestBase): def setUp(self): super(TestTransportAuth, self).setUp() - self.cfg = config.project('marconi') + self.cfg = cfg.ConfigOpts() def tearDown(self): super(TestTransportAuth, self).tearDown() - self.cfg.conf = cfg.ConfigOpts() def test_configs(self): - auth.strategy('keystone')._register_opts(self.cfg.conf) - self.assertIn('keystone_authtoken', self.cfg.conf) - self.assertIn('keystone_authtoken', dir(self.cfg.from_options())) + auth.strategy('keystone')._register_opts(self.cfg) + self.assertIn('keystone_authtoken', self.cfg) diff --git a/tests/unit/queues/transport/wsgi/test_claims.py b/tests/unit/queues/transport/wsgi/test_claims.py index 3fb7bf998..8fe3748bb 100644 --- a/tests/unit/queues/transport/wsgi/test_claims.py +++ b/tests/unit/queues/transport/wsgi/test_claims.py @@ -18,10 +18,10 @@ import uuid import ddt import falcon +from oslo.config import cfg from testtools import matchers import base # noqa -from marconi.common import config from marconi.openstack.common import timeutils from marconi import tests as testing @@ -32,8 +32,7 @@ class ClaimsBaseTest(base.TestBase): def setUp(self): super(ClaimsBaseTest, self).setUp() - self.wsgi_cfg = config.namespace( - 'queues:drivers:transport:wsgi').from_options() + self.wsgi_cfg = cfg.CONF['queues:drivers:transport:wsgi'] self.project_id = '480924' self.queue_path = '/v1/queues/fizbit' @@ -240,9 +239,7 @@ class ClaimsMongoDBTests(ClaimsBaseTest): def setUp(self): super(ClaimsMongoDBTests, self).setUp() - self.cfg = config.namespace( - 'queues:drivers:storage:mongodb' - ).from_options() + self.cfg = cfg.CONF['queues:drivers:storage:mongodb'] def tearDown(self): storage = self.boot.storage diff --git a/tests/unit/queues/transport/wsgi/test_messages.py b/tests/unit/queues/transport/wsgi/test_messages.py index 7864202d7..1a2670981 100644 --- a/tests/unit/queues/transport/wsgi/test_messages.py +++ b/tests/unit/queues/transport/wsgi/test_messages.py @@ -18,11 +18,11 @@ import uuid import ddt import falcon +from oslo.config import cfg import six from testtools import matchers import base # noqa -from marconi.common import config from marconi.openstack.common import timeutils from marconi.queues.transport import validation from marconi import tests as testing @@ -34,8 +34,7 @@ class MessagesBaseTest(base.TestBase): def setUp(self): super(MessagesBaseTest, self).setUp() - self.wsgi_cfg = config.namespace( - 'queues:drivers:transport:wsgi').from_options() + self.wsgi_cfg = cfg.CONF['queues:drivers:transport:wsgi'] self.project_id = '7e55e1a7e' self.queue_path = '/v1/queues/fizbit' diff --git a/tests/unit/queues/transport/wsgi/test_queue_lifecycle.py b/tests/unit/queues/transport/wsgi/test_queue_lifecycle.py index ea6d03973..7e1ac4d99 100644 --- a/tests/unit/queues/transport/wsgi/test_queue_lifecycle.py +++ b/tests/unit/queues/transport/wsgi/test_queue_lifecycle.py @@ -18,10 +18,10 @@ import json import ddt import falcon +from oslo.config import cfg import six import base # noqa -from marconi.common import config from marconi import tests as testing @@ -33,8 +33,7 @@ class QueueLifecycleBaseTest(base.TestBase): def setUp(self): super(QueueLifecycleBaseTest, self).setUp() - self.wsgi_cfg = config.namespace( - 'queues:drivers:transport:wsgi').from_options() + self.wsgi_cfg = cfg.CONF['queues:drivers:transport:wsgi'] def test_empty_project_id(self): path = '/v1/queues/gumshoe' @@ -317,9 +316,7 @@ class QueueLifecycleMongoDBTests(QueueLifecycleBaseTest): def setUp(self): super(QueueLifecycleMongoDBTests, self).setUp() - self.cfg = config.namespace( - 'queues:drivers:storage:mongodb' - ).from_options() + self.cfg = cfg.CONF['queues:drivers:storage:mongodb'] def tearDown(self): storage = self.boot.storage