From a7df08deb925b79a4eda6dbbd3a1ee1577d6ffff Mon Sep 17 00:00:00 2001 From: xywang <233652566@qq.com> Date: Sat, 28 Apr 2018 15:36:38 +0800 Subject: [PATCH] CONF file structure refactor This is the openstack governance goal that the config files should be merged into one folder to make the code to be better understood and managed. Change-Id: I33dd85145d9f5b0384323ffa51b4d68de0aeb5b7 --- doc/source/conf.py | 7 +- doc/source/configuration/zaqar.rst | 1 - doc/source/install/install-obs.rst | 2 +- doc/source/install/install-rdo.rst | 2 +- doc/source/install/install-ubuntu.rst | 2 +- etc/README-policy.json.sample | 2 +- etc/oslo-config-generator/zaqar.conf | 15 +- .../zaqar-policy-generator.conf | 0 ...nfiguration-refactor-0ff219ac59c96347.yaml | 6 + setup.cfg | 13 +- tox.ini | 2 +- zaqar/bootstrap.py | 7 +- zaqar/cmd/server.py | 4 +- zaqar/common/configs.py | 129 ------------- zaqar/conf/__init__.py | 74 ++++++++ zaqar/conf/default.py | 70 +++++++ zaqar/conf/drivers.py | 46 +++++ .../conf/drivers_management_store_mongodb.py | 157 ++++++++++++++++ zaqar/conf/drivers_management_store_redis.py | 86 +++++++++ .../drivers_management_store_sqlalchemy.py | 38 ++++ zaqar/conf/drivers_message_store_mongodb.py | 173 ++++++++++++++++++ zaqar/conf/drivers_message_store_redis.py | 86 +++++++++ zaqar/conf/drivers_message_store_swift.py | 80 ++++++++ zaqar/conf/drivers_transport_websocket.py | 59 ++++++ zaqar/conf/drivers_transport_wsgi.py | 39 ++++ zaqar/conf/notification.py | 93 ++++++++++ zaqar/conf/opts.py | 109 +++++++++++ zaqar/conf/pooling_catalog.py | 33 ++++ zaqar/conf/profiler.py | 45 +++++ zaqar/conf/signed_url.py | 32 ++++ zaqar/conf/storage.py | 61 ++++++ zaqar/conf/transport.py | 155 ++++++++++++++++ zaqar/storage/mongodb/driver.py | 18 +- zaqar/storage/mongodb/options.py | 147 --------------- zaqar/storage/pipeline.py | 26 +-- zaqar/storage/pooling.py | 18 +- zaqar/storage/redis/driver.py | 17 +- zaqar/storage/redis/options.py | 81 -------- zaqar/storage/sqlalchemy/driver.py | 10 +- zaqar/storage/sqlalchemy/options.py | 34 ---- zaqar/storage/swift/driver.py | 7 +- zaqar/storage/swift/options.py | 39 ---- zaqar/storage/utils.py | 4 +- zaqar/tests/base.py | 26 +-- zaqar/tests/functional/base.py | 2 - zaqar/tests/unit/common/storage/test_utils.py | 4 +- zaqar/tests/unit/common/test_decorators.py | 4 +- zaqar/tests/unit/storage/test_impl_mongodb.py | 21 ++- zaqar/tests/unit/storage/test_impl_redis.py | 8 +- zaqar/tests/unit/transport/websocket/base.py | 20 +- zaqar/tests/unit/transport/wsgi/base.py | 20 +- zaqar/transport/base.py | 41 +---- zaqar/transport/validation.py | 76 +------- zaqar/transport/websocket/driver.py | 33 +--- zaqar/transport/wsgi/driver.py | 21 +-- 55 files changed, 1573 insertions(+), 732 deletions(-) rename etc/{oslo-config-generator => }/zaqar-policy-generator.conf (100%) create mode 100644 releasenotes/notes/configuration-refactor-0ff219ac59c96347.yaml delete mode 100644 zaqar/common/configs.py create mode 100644 zaqar/conf/__init__.py create mode 100644 zaqar/conf/default.py create mode 100644 zaqar/conf/drivers.py create mode 100644 zaqar/conf/drivers_management_store_mongodb.py create mode 100644 zaqar/conf/drivers_management_store_redis.py create mode 100644 zaqar/conf/drivers_management_store_sqlalchemy.py create mode 100644 zaqar/conf/drivers_message_store_mongodb.py create mode 100644 zaqar/conf/drivers_message_store_redis.py create mode 100644 zaqar/conf/drivers_message_store_swift.py create mode 100644 zaqar/conf/drivers_transport_websocket.py create mode 100644 zaqar/conf/drivers_transport_wsgi.py create mode 100644 zaqar/conf/notification.py create mode 100644 zaqar/conf/opts.py create mode 100644 zaqar/conf/pooling_catalog.py create mode 100644 zaqar/conf/profiler.py create mode 100644 zaqar/conf/signed_url.py create mode 100644 zaqar/conf/storage.py create mode 100644 zaqar/conf/transport.py delete mode 100644 zaqar/storage/mongodb/options.py delete mode 100644 zaqar/storage/redis/options.py delete mode 100644 zaqar/storage/sqlalchemy/options.py delete mode 100644 zaqar/storage/swift/options.py diff --git a/doc/source/conf.py b/doc/source/conf.py index 6aa28c4a4..cbb9b56d4 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -48,10 +48,9 @@ extensions = ['sphinx.ext.coverage', 'openstackdocstheme', ] -config_generator_config_file = [ - ('../../etc/oslo-config-generator/zaqar.conf', - '_static/zaqar'), -] +config_generator_config_file = '../../etc/oslo-config-generator/zaqar.conf' +sample_config_basename = '_static/zaqar' + # autodoc generation is a bit aggressive and a nuisance # when doing heavy text edit cycles. Execute "export SPHINX_DEBUG=1" diff --git a/doc/source/configuration/zaqar.rst b/doc/source/configuration/zaqar.rst index c080b7f5e..29db5b569 100644 --- a/doc/source/configuration/zaqar.rst +++ b/doc/source/configuration/zaqar.rst @@ -6,4 +6,3 @@ zaqar.conf .. show-options:: :config-file: etc/oslo-config-generator/zaqar.conf - diff --git a/doc/source/install/install-obs.rst b/doc/source/install/install-obs.rst index 176520ff2..1a9d35402 100644 --- a/doc/source/install/install-obs.rst +++ b/doc/source/install/install-obs.rst @@ -278,7 +278,7 @@ Install and configure ``memcached``, ``uWSGI`` and Messaging on the web server .. code-block:: console - # oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf + # oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf # cp etc/zaqar.policy.yaml.sample /etc/zaqar/policy.yaml Edit any item as needed in policy.yaml. diff --git a/doc/source/install/install-rdo.rst b/doc/source/install/install-rdo.rst index 0f49a2f32..27a9ebbdf 100644 --- a/doc/source/install/install-rdo.rst +++ b/doc/source/install/install-rdo.rst @@ -278,7 +278,7 @@ Install and configure ``memcached``, ``uWSGI`` and Messaging on the web server .. code-block:: console - # oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf + # oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf # cp etc/zaqar.policy.yaml.sample /etc/zaqar/policy.yaml Edit any item as needed in policy.yaml. diff --git a/doc/source/install/install-ubuntu.rst b/doc/source/install/install-ubuntu.rst index 2ec14b271..0704026d0 100644 --- a/doc/source/install/install-ubuntu.rst +++ b/doc/source/install/install-ubuntu.rst @@ -266,7 +266,7 @@ Install and configure ``memcached``, ``uWSGI`` and Messaging on the web server .. code-block:: console - # oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf + # oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf # cp etc/zaqar.policy.yaml.sample /etc/zaqar/policy.yaml Edit any item as needed in policy.yaml. diff --git a/etc/README-policy.json.sample b/etc/README-policy.json.sample index 96bb1edc1..a8e0788ab 100644 --- a/etc/README-policy.json.sample +++ b/etc/README-policy.json.sample @@ -3,6 +3,6 @@ level of the zaqar directory: tox -egenpolicy Or run the command directly: - oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf + oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf The output file will be in etc folder, named "zaqar.policy.yaml.sample" by default. \ No newline at end of file diff --git a/etc/oslo-config-generator/zaqar.conf b/etc/oslo-config-generator/zaqar.conf index 0123ea44c..6b41d0073 100644 --- a/etc/oslo-config-generator/zaqar.conf +++ b/etc/oslo-config-generator/zaqar.conf @@ -1,18 +1,7 @@ [DEFAULT] -wrap_width = 80 +wrap_width = 79 output_file = etc/zaqar.conf.sample -namespace = zaqar.common.auth -namespace = zaqar.common.configs -namespace = zaqar.storage.pipeline -namespace = zaqar.storage.pooling -namespace = zaqar.storage.mongodb -namespace = zaqar.storage.redis -namespace = zaqar.storage.sqlalchemy -namespace = zaqar.storage.swift -namespace = zaqar.transport.wsgi -namespace = zaqar.transport.websocket -namespace = zaqar.transport.base -namespace = zaqar.transport.validation +namespace = zaqar namespace = keystonemiddleware.auth_token namespace = oslo.cache namespace = oslo.log diff --git a/etc/oslo-config-generator/zaqar-policy-generator.conf b/etc/zaqar-policy-generator.conf similarity index 100% rename from etc/oslo-config-generator/zaqar-policy-generator.conf rename to etc/zaqar-policy-generator.conf diff --git a/releasenotes/notes/configuration-refactor-0ff219ac59c96347.yaml b/releasenotes/notes/configuration-refactor-0ff219ac59c96347.yaml new file mode 100644 index 000000000..ee821b4d0 --- /dev/null +++ b/releasenotes/notes/configuration-refactor-0ff219ac59c96347.yaml @@ -0,0 +1,6 @@ +other: + - | + The code structure for configuration files are changed. This is insensitvie + for end users, but the persons who work for downstream changes should pay + attention. Please refactor your private configurations to ``zaqar/conf/`` + folder as well. diff --git a/setup.cfg b/setup.cfg index dbb4c45a6..59a9f496e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,18 +46,7 @@ zaqar.transport = websocket = zaqar.transport.websocket.driver:Driver oslo.config.opts = - zaqar.common.auth = zaqar.common.auth:_config_options - zaqar.common.configs = zaqar.common.configs:_config_options - zaqar.storage.pipeline = zaqar.storage.pipeline:_config_options - zaqar.storage.pooling = zaqar.storage.pooling:_config_options - zaqar.storage.mongodb = zaqar.storage.mongodb.options:_config_options - zaqar.storage.redis = zaqar.storage.redis.options:_config_options - zaqar.storage.sqlalchemy = zaqar.storage.sqlalchemy.options:_config_options - zaqar.storage.swift = zaqar.storage.swift.options:_config_options - zaqar.transport.wsgi = zaqar.transport.wsgi.driver:_config_options - zaqar.transport.websocket = zaqar.transport.websocket.driver:_config_options - zaqar.transport.base = zaqar.transport.base:_config_options - zaqar.transport.validation = zaqar.transport.validation:_config_options + zaqar = zaqar.conf.opts:list_opts zaqar.storage.stages = zaqar.notification.notifier = zaqar.notification.notifier:NotifierDriver diff --git a/tox.ini b/tox.ini index b57dddf35..22fa5c907 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,7 @@ commands = [testenv:genpolicy] commands = - oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf + oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf [testenv:cover] commands = diff --git a/zaqar/bootstrap.py b/zaqar/bootstrap.py index 91d47dcc2..bdf97cd5e 100644 --- a/zaqar/bootstrap.py +++ b/zaqar/bootstrap.py @@ -22,10 +22,11 @@ from stevedore import driver from zaqar.api import handler from zaqar.common import cache as oslo_cache -from zaqar.common import configs from zaqar.common import consts from zaqar.common import decorators from zaqar.common import errors +from zaqar.conf import drivers as driver_opts +from zaqar.conf import opts as opts_tool from zaqar.storage import pipeline from zaqar.storage import pooling from zaqar.storage import utils as storage_utils @@ -46,7 +47,7 @@ class Bootstrap(object): def __init__(self, conf): self.conf = conf - for group, opts in configs._config_options(): + for group, opts in opts_tool.list_opts_by_group(): self.conf.register_opts(opts, group=group) profiler_opts.set_defaults(self.conf) @@ -54,7 +55,7 @@ class Bootstrap(object): # wsgi. Websocket part will be added in the future. profile.setup(self.conf, 'Zaqar-server', socket.gethostname()) - self.driver_conf = self.conf[configs._DRIVER_GROUP] + self.driver_conf = self.conf[driver_opts.GROUP_NAME] @decorators.lazy_property(write=False) def api(self): diff --git a/zaqar/cmd/server.py b/zaqar/cmd/server.py index ae1673f34..55e04a89a 100644 --- a/zaqar/cmd/server.py +++ b/zaqar/cmd/server.py @@ -21,12 +21,12 @@ from oslo_reports import opts as gmr_opts from zaqar import bootstrap from zaqar.common import cli -from zaqar.common import configs +from zaqar.conf import default from zaqar import version # NOTE(eggmaster): define command line options for zaqar-server _CLI_OPTIONS = ( - configs._ADMIN_MODE_OPT, + default.admin_mode, cfg.BoolOpt('daemon', default=False, help='Run Zaqar server in the background.'), ) diff --git a/zaqar/common/configs.py b/zaqar/common/configs.py deleted file mode 100644 index 916e9a33e..000000000 --- a/zaqar/common/configs.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright (c) 2015 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from oslo_config import cfg - - -_ADMIN_MODE_OPT = cfg.BoolOpt('admin_mode', default=False, - help='Activate privileged endpoints.') - - -_GENERAL_OPTIONS = ( - _ADMIN_MODE_OPT, - cfg.BoolOpt('pooling', default=False, - help=('Enable pooling across multiple storage backends. ' - 'If pooling is enabled, the storage driver ' - 'configuration is used to determine where the ' - 'catalogue/control plane data is kept.'), - deprecated_opts=[cfg.DeprecatedOpt('sharding')]), - cfg.BoolOpt('unreliable', default=False, - help='Disable all reliability constraints.'), - cfg.ListOpt('enable_deprecated_api_versions', default=[], - item_type=cfg.types.List(item_type=cfg.types.String( - choices=('1', '1.1'))), - help='List of deprecated API versions to enable.'), - cfg.BoolOpt('enable_checksum', default=False, - help='Enable a checksum for message body. The default value ' - 'is False.'), -) - -_DRIVER_OPTIONS = ( - cfg.StrOpt('transport', default='wsgi', - help='Transport driver to use.'), - cfg.StrOpt('message_store', default='mongodb', - deprecated_opts=[cfg.DeprecatedOpt('storage')], - help='Storage driver to use as the messaging store.'), - cfg.StrOpt('management_store', default='mongodb', - help='Storage driver to use as the management store.'), -) - -_DRIVER_GROUP = 'drivers' - - -_SIGNED_URL_OPTIONS = ( - cfg.StrOpt('secret_key', - help=('Secret key used to encrypt pre-signed URLs.')), -) - -_SIGNED_URL_GROUP = 'signed_url' - - -_NOTIFICATION_OPTIONS = ( - cfg.StrOpt('smtp_command', default='/usr/sbin/sendmail -t -oi', - help=('The command of smtp to send email. The format is ' - '"command_name arg1 arg2".')), - cfg.IntOpt('max_notifier_workers', default=10, - help='The max amount of the notification workers.'), - cfg.BoolOpt('require_confirmation', default=False, - help='Whether the http/https/email subscription need to be ' - 'confirmed before notification.'), - cfg.StrOpt('external_confirmation_url', - help='The confirmation page url that will be used in email ' - 'subscription confirmation before notification.'), - cfg.DictOpt("subscription_confirmation_email_template", - default={'topic': 'Zaqar Notification - Subscription ' - 'Confirmation', - 'body': 'You have chosen to subscribe to the ' - 'queue: {0}. This queue belongs to ' - 'project: {1}. ' - 'To confirm this subscription, ' - 'click or visit this link below: {2}', - 'sender': 'Zaqar Notifications ' - ''}, - help="Defines the set of subscription confirmation email " - "content, including topic, body and sender. There is " - "a mapping is {0} -> queue name, {1} ->project id, " - "{2}-> confirm url in body string. User can use any of " - "the three value. But they can't use more than three."), - cfg.DictOpt("unsubscribe_confirmation_email_template", - default={'topic': 'Zaqar Notification - ' - 'Unsubscribe Confirmation', - 'body': 'You have unsubscribed successfully to the ' - 'queue: {0}. This queue belongs to ' - 'project: {1}. ' - 'To resubscribe this subscription, ' - 'click or visit this link below: {2}', - 'sender': 'Zaqar Notifications ' - ''}, - help="Defines the set of unsubscribe confirmation email " - "content, including topic, body and sender. There is " - "a mapping is {0} -> queue name, {1} ->project id, " - "{2}-> confirm url in body string. User can use any of " - "the three value. But they can't use more than three."), -) - -_NOTIFICATION_GROUP = 'notification' - - -_PROFILER_OPTIONS = [ - cfg.BoolOpt("trace_wsgi_transport", default=False, - help="If False doesn't trace any transport requests." - "Please note that it doesn't work for websocket now."), - cfg.BoolOpt("trace_message_store", default=False, - help="If False doesn't trace any message store requests."), - cfg.BoolOpt("trace_management_store", default=False, - help="If False doesn't trace any management store requests.") -] - -_PROFILER_GROUP = "profiler" - - -def _config_options(): - return [(None, _GENERAL_OPTIONS), - (_DRIVER_GROUP, _DRIVER_OPTIONS), - (_SIGNED_URL_GROUP, _SIGNED_URL_OPTIONS), - (_NOTIFICATION_GROUP, _NOTIFICATION_OPTIONS), - (_PROFILER_GROUP, _PROFILER_OPTIONS)] diff --git a/zaqar/conf/__init__.py b/zaqar/conf/__init__.py new file mode 100644 index 000000000..d565c4be0 --- /dev/null +++ b/zaqar/conf/__init__.py @@ -0,0 +1,74 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from oslo_cache import core as cache +from oslo_config import cfg +from oslo_log import log + +from zaqar.conf import default +from zaqar.conf import drivers +from zaqar.conf import drivers_management_store_mongodb +from zaqar.conf import drivers_management_store_redis +from zaqar.conf import drivers_management_store_sqlalchemy +from zaqar.conf import drivers_message_store_mongodb +from zaqar.conf import drivers_message_store_redis +from zaqar.conf import drivers_message_store_swift +from zaqar.conf import drivers_transport_websocket +from zaqar.conf import drivers_transport_wsgi +from zaqar.conf import notification +from zaqar.conf import pooling_catalog +from zaqar.conf import profiler +from zaqar.conf import signed_url +from zaqar.conf import storage +from zaqar.conf import transport + +CONF = cfg.CONF + + +conf_modules = [ + default, + drivers, + drivers_management_store_mongodb, + drivers_management_store_redis, + drivers_management_store_sqlalchemy, + drivers_message_store_mongodb, + drivers_message_store_redis, + drivers_message_store_swift, + drivers_transport_websocket, + drivers_transport_wsgi, + notification, + pooling_catalog, + profiler, + signed_url, + storage, + transport +] + + +def setup_logging(): + """Set up logging for the keystone package.""" + log.setup(CONF, 'zaqar') + logging.captureWarnings(True) + + +def configure(conf=None): + if conf is None: + conf = CONF + + for module in conf_modules: + module.register_opts(conf) + + # add oslo.cache related config options + cache.configure(conf) diff --git a/zaqar/conf/default.py b/zaqar/conf/default.py new file mode 100644 index 000000000..15c31903f --- /dev/null +++ b/zaqar/conf/default.py @@ -0,0 +1,70 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +admin_mode = cfg.BoolOpt( + 'admin_mode', default=False, + help='Activate privileged endpoints.') + + +pooling = cfg.BoolOpt( + 'pooling', default=False, + help=('Enable pooling across multiple storage backends. ' + 'If pooling is enabled, the storage driver ' + 'configuration is used to determine where the ' + 'catalogue/control plane data is kept.'), + deprecated_opts=[cfg.DeprecatedOpt('sharding')]) + + +unreliable = cfg.BoolOpt( + 'unreliable', default=False, + help='Disable all reliability constraints.') + + +enable_deprecated_api_versions = cfg.ListOpt( + 'enable_deprecated_api_versions', default=[], + item_type=cfg.types.List(item_type=cfg.types.String(choices=('1', '1.1'))), + help='List of deprecated API versions to enable.') + + +enable_checksum = cfg.BoolOpt( + 'enable_checksum', default=False, + help='Enable a checksum for message body. The default value is False.') + + +auth_strategy = cfg.StrOpt( + 'auth_strategy', default='', + help=('Backend to use for authentication. ' + 'For no auth, keep it empty. ' + 'Existing strategies: keystone. ' + 'See also the keystone_authtoken section below')) + +GROUP_NAME = 'DEFAULT' +ALL_OPTS = [ + admin_mode, + pooling, + unreliable, + enable_deprecated_api_versions, + enable_checksum, + auth_strategy +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers.py b/zaqar/conf/drivers.py new file mode 100644 index 000000000..3110ef243 --- /dev/null +++ b/zaqar/conf/drivers.py @@ -0,0 +1,46 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +transport = cfg.StrOpt( + 'transport', default='wsgi', + help='Transport driver to use.') + + +message_store = cfg.StrOpt( + 'message_store', default='mongodb', + deprecated_opts=[cfg.DeprecatedOpt('storage')], + help='Storage driver to use as the messaging store.') + + +management_store = cfg.StrOpt( + 'management_store', default='mongodb', + help='Storage driver to use as the management store.') + + +GROUP_NAME = 'drivers' +ALL_OPTS = [ + transport, + message_store, + management_store +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_management_store_mongodb.py b/zaqar/conf/drivers_management_store_mongodb.py new file mode 100644 index 000000000..782fe7789 --- /dev/null +++ b/zaqar/conf/drivers_management_store_mongodb.py @@ -0,0 +1,157 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +_deprecated_group = 'drivers:storage:mongodb' + + +ssl_keyfile = cfg.StrOpt( + 'ssl_keyfile', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_keyfile', + group=_deprecated_group), ], + help='The private keyfile used to identify the local ' + 'connection against mongod. If included with ' + 'the ``certifle`` then only the ``ssl_certfile``' + ' is needed.') + + +ssl_certfile = cfg.StrOpt( + 'ssl_certfile', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_certfile', + group=_deprecated_group), ], + help='The certificate file used to identify the ' + 'local connection against mongod.') + + +ssl_cert_reqs = cfg.StrOpt( + 'ssl_cert_reqs', default='CERT_REQUIRED', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_cert_reqs', + group=_deprecated_group), ], + help='Specifies whether a certificate is required ' + 'from the other side of the connection, and ' + 'whether it will be validated if provided. It ' + 'must be one of the three values ``CERT_NONE``' + '(certificates ignored), ``CERT_OPTIONAL``' + '(not required, but validated if provided), or' + ' ``CERT_REQUIRED``(required and validated). ' + 'If the value of this parameter is not ' + '``CERT_NONE``, then the ``ssl_ca_cert`` ' + 'parameter must point to a file of CA ' + 'certificates.') + + +ssl_ca_certs = cfg.StrOpt( + 'ssl_ca_certs', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_ca_certs', + group=_deprecated_group), ], + help='The ca_certs file contains a set of concatenated ' + '"certification authority" certificates, which are ' + 'used to validate certificates passed from the other ' + 'end of the connection.') + + +uri = cfg.StrOpt( + 'uri', + deprecated_opts=[cfg.DeprecatedOpt( + 'uri', + group=_deprecated_group), ], + help='Mongodb Connection URI. If ssl connection enabled, ' + 'then ``ssl_keyfile``, ``ssl_certfile``, ' + '``ssl_cert_reqs``, ``ssl_ca_certs`` need to be set ' + 'accordingly.') + + +database = cfg.StrOpt( + 'database', default='zaqar', + deprecated_opts=[cfg.DeprecatedOpt( + 'database', + group=_deprecated_group), ], + help='Database name.') + +max_attempts = cfg.IntOpt( + 'max_attempts', min=0, default=1000, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_attempts', + group=_deprecated_group), ], + help=('Maximum number of times to retry a failed operation. ' + 'Currently only used for retrying a message post.')) + + +max_retry_sleep = cfg.FloatOpt( + 'max_retry_sleep', default=0.1, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_retry_sleep', + group=_deprecated_group), ], + help=('Maximum sleep interval between retries ' + '(actual sleep time increases linearly ' + 'according to number of attempts performed).')) + + +max_retry_jitter = cfg.FloatOpt( + 'max_retry_jitter', default=0.005, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_retry_jitter', + group=_deprecated_group), ], + help=('Maximum jitter interval, to be added to the ' + 'sleep interval, in order to decrease probability ' + 'that parallel requests will retry at the ' + 'same instant.')) + + +max_reconnect_attempts = cfg.IntOpt( + 'max_reconnect_attempts', default=10, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_reconnect_attempts', + group=_deprecated_group), ], + help=('Maximum number of times to retry an operation that ' + 'failed due to a primary node failover.')) + + +reconnect_sleep = cfg.FloatOpt( + 'reconnect_sleep', default=0.020, + deprecated_opts=[cfg.DeprecatedOpt( + 'reconnect_sleep', + group=_deprecated_group), ], + help=('Base sleep interval between attempts to reconnect ' + 'after a primary node failover. ' + 'The actual sleep time increases exponentially (power ' + 'of 2) each time the operation is retried.')) + + +GROUP_NAME = 'drivers:management_store:mongodb' +ALL_OPTS = [ + ssl_keyfile, + ssl_certfile, + ssl_cert_reqs, + ssl_ca_certs, + uri, + database, + max_attempts, + max_retry_sleep, + max_retry_jitter, + max_reconnect_attempts, + reconnect_sleep +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_management_store_redis.py b/zaqar/conf/drivers_management_store_redis.py new file mode 100644 index 000000000..d6be0c6cf --- /dev/null +++ b/zaqar/conf/drivers_management_store_redis.py @@ -0,0 +1,86 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +_deprecated_group = 'drivers:storage:redis' + + +uri = cfg.StrOpt( + 'uri', default="redis://127.0.0.1:6379", + deprecated_opts=[cfg.DeprecatedOpt( + 'uri', + group=_deprecated_group), ], + help=('Redis connection URI, taking one of three forms. ' + 'For a direct connection to a Redis server, use ' + 'the form "redis://[:password]@host[:port][?options]", ' + 'where password is redis-server\'s password, when' + 'redis-server is set password, the password option' + 'needs to be set. port defaults to 6379 if not' + 'specified. For an HA master-slave Redis cluster using' + ' Redis Sentinel, use the form ' + '"redis://[:password]@host1[:port1]' + '[,host2[:port2],...,hostN[:portN]][?options]", ' + 'where each host specified corresponds to an ' + 'instance of redis-sentinel. In this form, the ' + 'name of the Redis master used in the Sentinel ' + 'configuration must be included in the query ' + 'string as "master=". Finally, to connect ' + 'to a local instance of Redis over a unix socket, ' + 'you may use the form ' + '"redis:[:password]@/path/to/redis.sock[?options]".' + ' In all forms, the "socket_timeout" option may be' + 'specified in the query string. Its value is ' + 'given in seconds. If not provided, ' + '"socket_timeout" defaults to 0.1 seconds.' + 'There are multiple database instances in redis ' + 'database, for example in the /etc/redis/redis.conf, ' + 'if the parameter is "database 16", there are 16 ' + 'database instances. By default, the data is stored ' + 'in db = 0 database, if you want to use db = 1 ' + 'database, you can use the following form: ' + '"redis://host[:port][?dbid=1]".')) + + +max_reconnect_attempts = cfg.IntOpt( + 'max_reconnect_attempts', default=10, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_reconnect_attempts', + group=_deprecated_group), ], + help=('Maximum number of times to retry an operation that ' + 'failed due to a redis node failover.')) + + +reconnect_sleep = cfg.FloatOpt( + 'reconnect_sleep', default=1.0, + deprecated_opts=[cfg.DeprecatedOpt( + 'reconnect_sleep', + group=_deprecated_group), ], + help=('Base sleep interval between attempts to reconnect ' + 'after a redis node failover. ')) + + +GROUP_NAME = 'drivers:management_store:redis' +ALL_OPTS = [ + uri, + max_reconnect_attempts, + reconnect_sleep +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_management_store_sqlalchemy.py b/zaqar/conf/drivers_management_store_sqlalchemy.py new file mode 100644 index 000000000..3de3feb25 --- /dev/null +++ b/zaqar/conf/drivers_management_store_sqlalchemy.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +_deprecated_group = 'drivers:storage:sqlalchemy' + +uri = cfg.StrOpt( + 'uri', default='sqlite:///:memory:', + deprecated_opts=[cfg.DeprecatedOpt( + 'uri', + group=_deprecated_group), ], + help='An sqlalchemy URL') + + +GROUP_NAME = 'drivers:management_store:sqlalchemy' +ALL_OPTS = [ + uri +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_message_store_mongodb.py b/zaqar/conf/drivers_message_store_mongodb.py new file mode 100644 index 000000000..314023a35 --- /dev/null +++ b/zaqar/conf/drivers_message_store_mongodb.py @@ -0,0 +1,173 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +_deprecated_group = 'drivers:storage:mongodb' + + +ssl_keyfile = cfg.StrOpt( + 'ssl_keyfile', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_keyfile', + group=_deprecated_group), ], + help='The private keyfile used to identify the local ' + 'connection against mongod. If included with ' + 'the ``certifle`` then only the ``ssl_certfile``' + ' is needed.') + + +ssl_certfile = cfg.StrOpt( + 'ssl_certfile', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_certfile', + group=_deprecated_group), ], + help='The certificate file used to identify the ' + 'local connection against mongod.') + + +ssl_cert_reqs = cfg.StrOpt( + 'ssl_cert_reqs', default='CERT_REQUIRED', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_cert_reqs', + group=_deprecated_group), ], + help='Specifies whether a certificate is required ' + 'from the other side of the connection, and ' + 'whether it will be validated if provided. It ' + 'must be one of the three values ``CERT_NONE``' + '(certificates ignored), ``CERT_OPTIONAL``' + '(not required, but validated if provided), or' + ' ``CERT_REQUIRED``(required and validated). ' + 'If the value of this parameter is not ' + '``CERT_NONE``, then the ``ssl_ca_cert`` ' + 'parameter must point to a file of CA ' + 'certificates.') + + +ssl_ca_certs = cfg.StrOpt( + 'ssl_ca_certs', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_ca_certs', + group=_deprecated_group), ], + help='The ca_certs file contains a set of concatenated ' + '"certification authority" certificates, which are ' + 'used to validate certificates passed from the other ' + 'end of the connection.') + + +uri = cfg.StrOpt( + 'uri', + deprecated_opts=[cfg.DeprecatedOpt( + 'uri', + group=_deprecated_group), ], + help='Mongodb Connection URI. If ssl connection enabled, ' + 'then ``ssl_keyfile``, ``ssl_certfile``, ' + '``ssl_cert_reqs``, ``ssl_ca_certs`` need to be set ' + 'accordingly.') + + +database = cfg.StrOpt( + 'database', default='zaqar', + deprecated_opts=[cfg.DeprecatedOpt( + 'database', + group=_deprecated_group), ], + help='Database name.') + +max_attempts = cfg.IntOpt( + 'max_attempts', min=0, default=1000, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_attempts', + group=_deprecated_group), ], + help=('Maximum number of times to retry a failed operation. ' + 'Currently only used for retrying a message post.')) + + +max_retry_sleep = cfg.FloatOpt( + 'max_retry_sleep', default=0.1, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_retry_sleep', + group=_deprecated_group), ], + help=('Maximum sleep interval between retries ' + '(actual sleep time increases linearly ' + 'according to number of attempts performed).')) + + +max_retry_jitter = cfg.FloatOpt( + 'max_retry_jitter', default=0.005, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_retry_jitter', + group=_deprecated_group), ], + help=('Maximum jitter interval, to be added to the ' + 'sleep interval, in order to decrease probability ' + 'that parallel requests will retry at the ' + 'same instant.')) + + +max_reconnect_attempts = cfg.IntOpt( + 'max_reconnect_attempts', default=10, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_reconnect_attempts', + group=_deprecated_group), ], + help=('Maximum number of times to retry an operation that ' + 'failed due to a primary node failover.')) + + +reconnect_sleep = cfg.FloatOpt( + 'reconnect_sleep', default=0.020, + deprecated_opts=[cfg.DeprecatedOpt( + 'reconnect_sleep', + group=_deprecated_group), ], + help=('Base sleep interval between attempts to reconnect ' + 'after a primary node failover. ' + 'The actual sleep time increases exponentially (power ' + 'of 2) each time the operation is retried.')) + + +partitions = cfg.IntOpt( + 'partitions', default=2, + deprecated_opts=[cfg.DeprecatedOpt( + 'partitions', + group=_deprecated_group), ], + help=('Number of databases across which to ' + 'partition message data, in order to ' + 'reduce writer lock %. DO NOT change ' + 'this setting after initial deployment. ' + 'It MUST remain static. Also, you ' + 'should not need a large number of partitions ' + 'to improve performance, esp. if deploying ' + 'MongoDB on SSD storage.')) + + +GROUP_NAME = 'drivers:message_store:mongodb' +ALL_OPTS = [ + ssl_keyfile, + ssl_certfile, + ssl_cert_reqs, + ssl_ca_certs, + uri, + database, + max_attempts, + max_retry_sleep, + max_retry_jitter, + max_reconnect_attempts, + reconnect_sleep, + partitions +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_message_store_redis.py b/zaqar/conf/drivers_message_store_redis.py new file mode 100644 index 000000000..1584d8e18 --- /dev/null +++ b/zaqar/conf/drivers_message_store_redis.py @@ -0,0 +1,86 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +_deprecated_group = 'drivers:storage:redis' + + +uri = cfg.StrOpt( + 'uri', default="redis://127.0.0.1:6379", + deprecated_opts=[cfg.DeprecatedOpt( + 'uri', + group=_deprecated_group), ], + help=('Redis connection URI, taking one of three forms. ' + 'For a direct connection to a Redis server, use ' + 'the form "redis://[:password]@host[:port][?options]", ' + 'where password is redis-server\'s password, when' + 'redis-server is set password, the password option' + 'needs to be set. port defaults to 6379 if not' + 'specified. For an HA master-slave Redis cluster using' + ' Redis Sentinel, use the form ' + '"redis://[:password]@host1[:port1]' + '[,host2[:port2],...,hostN[:portN]][?options]", ' + 'where each host specified corresponds to an ' + 'instance of redis-sentinel. In this form, the ' + 'name of the Redis master used in the Sentinel ' + 'configuration must be included in the query ' + 'string as "master=". Finally, to connect ' + 'to a local instance of Redis over a unix socket, ' + 'you may use the form ' + '"redis:[:password]@/path/to/redis.sock[?options]".' + ' In all forms, the "socket_timeout" option may be' + 'specified in the query string. Its value is ' + 'given in seconds. If not provided, ' + '"socket_timeout" defaults to 0.1 seconds.' + 'There are multiple database instances in redis ' + 'database, for example in the /etc/redis/redis.conf, ' + 'if the parameter is "database 16", there are 16 ' + 'database instances. By default, the data is stored ' + 'in db = 0 database, if you want to use db = 1 ' + 'database, you can use the following form: ' + '"redis://host[:port][?dbid=1]".')) + + +max_reconnect_attempts = cfg.IntOpt( + 'max_reconnect_attempts', default=10, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_reconnect_attempts', + group=_deprecated_group), ], + help=('Maximum number of times to retry an operation that ' + 'failed due to a redis node failover.')) + + +reconnect_sleep = cfg.FloatOpt( + 'reconnect_sleep', default=1.0, + deprecated_opts=[cfg.DeprecatedOpt( + 'reconnect_sleep', + group=_deprecated_group), ], + help=('Base sleep interval between attempts to reconnect ' + 'after a redis node failover. ')) + + +GROUP_NAME = 'drivers:message_store:redis' +ALL_OPTS = [ + uri, + max_reconnect_attempts, + reconnect_sleep +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_message_store_swift.py b/zaqar/conf/drivers_message_store_swift.py new file mode 100644 index 000000000..f9e2d0a3a --- /dev/null +++ b/zaqar/conf/drivers_message_store_swift.py @@ -0,0 +1,80 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +auth_url = cfg.StrOpt( + "auth_url", default="http://127.0.0.1:5000/v3/", + help="URI of Keystone endpoint to discover Swift") + + +uri = cfg.StrOpt( + "uri", + default="swift://demo:nomoresecrete@/demo", + help="Custom URI describing the swift connection.") + + +insecure = cfg.StrOpt( + "insecure", default=False, + help="Don't check SSL certificate") + + +project_domain_id = cfg.StrOpt( + "project_domain_id", default="default", + help="Domain ID containing project") + + +project_domain_name = cfg.StrOpt( + "project_domain_name", + help="Domain name containing project") + + +user_domain_id = cfg.StrOpt( + "user_domain_id", default="default", + help="User's domain id") + + +user_domain_name = cfg.StrOpt( + "user_domain_name", help="User's domain name") + + +region_name = cfg.StrOpt( + "region_name", help="Region name") + + +interface = cfg.StrOpt( + "interface", default="publicURL", + help="The default interface for endpoint URL " + "discovery.") + + +GROUP_NAME = 'drivers:message_store:swift' +ALL_OPTS = [ + auth_url, + uri, + insecure, + project_domain_id, + project_domain_name, + user_domain_id, + user_domain_name, + region_name, + interface +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_transport_websocket.py b/zaqar/conf/drivers_transport_websocket.py new file mode 100644 index 000000000..ed66c701d --- /dev/null +++ b/zaqar/conf/drivers_transport_websocket.py @@ -0,0 +1,59 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +bind = cfg.HostAddressOpt( + 'bind', default='127.0.0.1', + help='Address on which the self-hosting server will ' + 'listen.') + + +port = cfg.PortOpt( + 'port', default=9000, + help='Port on which the self-hosting server will listen.') + + +external_port = cfg.PortOpt( + 'external-port', + help='Port on which the service is provided to the user.') + + +notification_bind = cfg.HostAddressOpt( + 'notification-bind', + help='Address on which the notification server will ' + 'listen.') + + +notification_port = cfg.PortOpt( + 'notification-port', default=0, + help='Port on which the notification server will listen.') + + +GROUP_NAME = 'drivers:transport:websocket' +ALL_OPTS = [ + bind, + port, + external_port, + notification_bind, + notification_port +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_transport_wsgi.py b/zaqar/conf/drivers_transport_wsgi.py new file mode 100644 index 000000000..95e13d7bb --- /dev/null +++ b/zaqar/conf/drivers_transport_wsgi.py @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +bind = cfg.HostAddressOpt( + 'bind', default='127.0.0.1', + help='Address on which the self-hosting server will ' + 'listen.') + + +port = cfg.PortOpt( + 'port', default=8888, + help='Port on which the self-hosting server will listen.') + + +GROUP_NAME = 'drivers:transport:wsgi' +ALL_OPTS = [ + bind, + port +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/notification.py b/zaqar/conf/notification.py new file mode 100644 index 000000000..38653ce1b --- /dev/null +++ b/zaqar/conf/notification.py @@ -0,0 +1,93 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +smtp_command = cfg.StrOpt( + 'smtp_command', default='/usr/sbin/sendmail -t -oi', + help=( + 'The command of smtp to send email. The format is ' + '"command_name arg1 arg2".')) + + +max_notifier_workers = cfg.IntOpt( + 'max_notifier_workers', default=10, + help='The max amount of the notification workers.') + + +require_confirmation = cfg.BoolOpt( + 'require_confirmation', default=False, + help='Whether the http/https/email subscription need to be confirmed ' + 'before notification.') + + +external_confirmation_url = cfg.StrOpt( + 'external_confirmation_url', + help='The confirmation page url that will be used in email subscription ' + 'confirmation before notification.') + + +subscription_confirmation_email_template = cfg.DictOpt( + "subscription_confirmation_email_template", + default={'topic': 'Zaqar Notification - Subscription ' + 'Confirmation', + 'body': 'You have chosen to subscribe to the ' + 'queue: {0}. This queue belongs to ' + 'project: {1}. ' + 'To confirm this subscription, ' + 'click or visit this link below: {2}', + 'sender': 'Zaqar Notifications ' + ''}, + help="Defines the set of subscription confirmation email content, " + "including topic, body and sender. There is a mapping is " + "{0} -> queue name, {1} ->project id, {2}-> confirm url in body " + "string. User can use any of the three values. But they can't use " + "more than three.") + + +unsubscribe_confirmation_email_template = cfg.DictOpt( + "unsubscribe_confirmation_email_template", + default={'topic': 'Zaqar Notification - ' + 'Unsubscribe Confirmation', + 'body': 'You have unsubscribed successfully to the ' + 'queue: {0}. This queue belongs to ' + 'project: {1}. ' + 'To resubscribe this subscription, ' + 'click or visit this link below: {2}', + 'sender': 'Zaqar Notifications ' + ''}, + help="Defines the set of unsubscribe confirmation email content, " + "including topic, body and sender. There is a mapping is " + "{0} -> queue name, {1} ->project id, {2}-> confirm url in body " + "string. User can use any of the three values. But they can't use " + "more than three.") + + +GROUP_NAME = 'notification' +ALL_OPTS = [ + smtp_command, + max_notifier_workers, + require_confirmation, + external_confirmation_url, + subscription_confirmation_email_template, + unsubscribe_confirmation_email_template +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/opts.py b/zaqar/conf/opts.py new file mode 100644 index 000000000..e826f4af9 --- /dev/null +++ b/zaqar/conf/opts.py @@ -0,0 +1,109 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Single point of entry to generate the sample configuration file. + +This module collects all the necessary info from the other modules in this +package. It is assumed that: + +* Every other module in this package has a 'list_opts' function which + returns a dict where: + + * The keys are strings which are the group names. + + * The value of each key is a list of config options for that group. + +* The conf package doesn't have further packages with config options. + +* This module is only used in the context of sample file generation. + +""" + +import collections +import importlib +import os +import pkgutil + + +LIST_OPTS_FUNC_NAME = 'list_opts' +IGNORED_MODULES = ('opts', 'constants', 'utils') + + +def list_opts(): + opts = collections.defaultdict(list) + module_names = _list_module_names() + imported_modules = _import_modules(module_names) + _append_config_options(imported_modules, opts) + return _tupleize(opts) + + +def list_opts_by_group(): + opts = [] + module_names = _list_module_names() + imported_modules = _import_modules(module_names) + for module in imported_modules: + configs = module.list_opts() + group_name = (module.GROUP_NAME if module.GROUP_NAME != 'DEFAULT' + else None) + opts.append((group_name, configs[module.GROUP_NAME])) + return opts + + +def _tupleize(d): + """Convert a dict of options to the 2-tuple format.""" + return [(key, value) for key, value in d.items()] + + +def _list_module_names(): + module_names = [] + package_path = os.path.dirname(os.path.abspath(__file__)) + for _, module_name, ispkg in pkgutil.iter_modules(path=[package_path]): + if module_name in IGNORED_MODULES or ispkg: + # Skip this module. + continue + else: + module_names.append(module_name) + return module_names + + +def _import_modules(module_names): + imported_modules = [] + for module_name in module_names: + full_module_path = '.'.join(__name__.split('.')[:-1] + [module_name]) + module = importlib.import_module(full_module_path) + if not hasattr(module, LIST_OPTS_FUNC_NAME): + raise Exception( + "The module '%s' should have a '%s' function which " + "returns the config options." % ( + full_module_path, + LIST_OPTS_FUNC_NAME)) + else: + imported_modules.append(module) + return imported_modules + + +def _process_old_opts(configs): + """Convert old-style 2-tuple configs to dicts.""" + if isinstance(configs, tuple): + configs = [configs] + return {label: options for label, options in configs} + + +def _append_config_options(imported_modules, config_options): + for module in imported_modules: + configs = module.list_opts() + # TODO(markus_z): Remove this compatibility shim once all list_opts() + # functions have been updated to return dicts. + if not isinstance(configs, dict): + configs = _process_old_opts(configs) + for key, val in configs.items(): + config_options[key].extend(val) diff --git a/zaqar/conf/pooling_catalog.py b/zaqar/conf/pooling_catalog.py new file mode 100644 index 000000000..32beff5b0 --- /dev/null +++ b/zaqar/conf/pooling_catalog.py @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +enable_virtual_pool = cfg.BoolOpt( + 'enable_virtual_pool', default=False, + help='If enabled, the message_store will be used as the storage for the ' + 'virtual pool.') + + +GROUP_NAME = 'pooling:catalog' +ALL_OPTS = [ + enable_virtual_pool +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/profiler.py b/zaqar/conf/profiler.py new file mode 100644 index 000000000..87a1124e4 --- /dev/null +++ b/zaqar/conf/profiler.py @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +trace_wsgi_transport = cfg.BoolOpt( + "trace_wsgi_transport", default=False, + help="If False doesn't trace any transport requests." + "Please note that it doesn't work for websocket now.") + + +trace_message_store = cfg.BoolOpt( + "trace_message_store", default=False, + help="If False doesn't trace any message store requests.") + + +trace_management_store = cfg.BoolOpt( + "trace_management_store", default=False, + help="If False doesn't trace any management store requests.") + + +GROUP_NAME = 'profiler' +ALL_OPTS = [ + trace_wsgi_transport, + trace_message_store, + trace_management_store +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/signed_url.py b/zaqar/conf/signed_url.py new file mode 100644 index 000000000..151b8057d --- /dev/null +++ b/zaqar/conf/signed_url.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +secret_key = cfg.StrOpt('secret_key', + help='Secret key used to encrypt pre-signed URLs.') + + +GROUP_NAME = 'signed_url' +ALL_OPTS = [ + secret_key +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/storage.py b/zaqar/conf/storage.py new file mode 100644 index 000000000..2779e3c83 --- /dev/null +++ b/zaqar/conf/storage.py @@ -0,0 +1,61 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +from zaqar.i18n import _ + + +queue_pipeline = cfg.ListOpt( + 'queue_pipeline', default=[], + help=_('Pipeline to use for processing queue operations. This pipeline ' + 'will be consumed before calling the storage driver\'s controller ' + 'methods.')) + + +message_pipeline = cfg.ListOpt( + 'message_pipeline', default=[], + help=_('Pipeline to use for processing message operations. This pipeline ' + 'will be consumed before calling the storage driver\'s controller ' + 'methods.')) + + +claim_pipeline = cfg.ListOpt( + 'claim_pipeline', default=[], + help=_('Pipeline to use for processing claim operations. This pipeline ' + 'will be consumed before calling the storage driver\'s controller ' + 'methods.')) + + +subscription_pipeline = cfg.ListOpt( + 'subscription_pipeline', default=[], + help=_('Pipeline to use for processing subscription operations. This ' + 'pipeline will be consumed before calling the storage driver\'s ' + 'controller methods.')) + + +GROUP_NAME = 'storage' +ALL_OPTS = [ + queue_pipeline, + message_pipeline, + claim_pipeline, + subscription_pipeline +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/transport.py b/zaqar/conf/transport.py new file mode 100644 index 000000000..0a5fd71a9 --- /dev/null +++ b/zaqar/conf/transport.py @@ -0,0 +1,155 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +default_message_ttl = cfg.IntOpt( + 'default_message_ttl', default=3600, + help='Defines how long a message will be accessible.') + + +default_message_delay = cfg.IntOpt( + 'default_message_delay', default=0, + help=('Defines the defautl value for queue delay seconds.' + 'The 0 means the delayed queues feature is close.')) + + +default_claim_ttl = cfg.IntOpt( + 'default_claim_ttl', default=300, + help='Defines how long a message will be in claimed state.') + + +default_claim_grace = cfg.IntOpt( + 'default_claim_grace', default=60, + help='Defines the message grace period in seconds.') + + +default_subscription_ttl = cfg.IntOpt( + 'default_subscription_ttl', default=3600, + help='Defines how long a subscription will be available.') + + +max_queues_per_page = cfg.IntOpt( + 'max_queues_per_page', default=20, + deprecated_name='queue_paging_uplimit', + deprecated_group='limits:transport', + help='Defines the maximum number of queues per page.') + + +max_messages_per_page = cfg.IntOpt( + 'max_messages_per_page', default=20, + deprecated_name='message_paging_uplimit', + deprecated_group='limits:transport', + help='Defines the maximum number of messages per page.') + + +max_subscriptions_per_page = cfg.IntOpt( + 'max_subscriptions_per_page', default=20, + deprecated_name='subscription_paging_uplimit', + deprecated_group='limits:transport', + help='Defines the maximum number of subscriptions per page.') + + +max_messages_per_claim_or_pop = cfg.IntOpt( + 'max_messages_per_claim_or_pop', default=20, + deprecated_name='max_messages_per_claim', + help='The maximum number of messages that can be claimed (OR) ' + 'popped in a single request') + + +max_queue_metadata = cfg.IntOpt( + 'max_queue_metadata', default=64 * 1024, + deprecated_name='metadata_size_uplimit', + deprecated_group='limits:transport', + help='Defines the maximum amount of metadata in a queue.') + + +max_messages_post_size = cfg.IntOpt( + 'max_messages_post_size', default=256 * 1024, + deprecated_name='message_size_uplimit', + deprecated_group='limits:transport', + deprecated_opts=[cfg.DeprecatedOpt('max_message_size')], + help='Defines the maximum size of message posts.') + + +max_message_ttl = cfg.IntOpt( + 'max_message_ttl', default=1209600, + deprecated_name='message_ttl_max', + deprecated_group='limits:transport', + help='Maximum amount of time a message will be available.') + + +max_message_delay = cfg.IntOpt( + 'max_message_delay', default=900, + help='Maximum delay seconds for messages can be claimed.') + + +max_claim_ttl = cfg.IntOpt( + 'max_claim_ttl', default=43200, + deprecated_name='claim_ttl_max', + deprecated_group='limits:transport', + help='Maximum length of a message in claimed state.') + + +max_claim_grace = cfg.IntOpt( + 'max_claim_grace', default=43200, + deprecated_name='claim_grace_max', + deprecated_group='limits:transport', + help='Defines the maximum message grace period in seconds.') + + +subscriber_types = cfg.ListOpt( + 'subscriber_types', default=['http', 'https', 'mailto', + 'trust+http', 'trust+https'], + help='Defines supported subscriber types.') + + +max_flavors_per_page = cfg.IntOpt( + 'max_flavors_per_page', default=20, + help='Defines the maximum number of flavors per page.') + + +max_pools_per_page = cfg.IntOpt( + 'max_pools_per_page', default=20, + help='Defines the maximum number of pools per page.') + + +GROUP_NAME = 'transport' +ALL_OPTS = [ + default_message_ttl, + default_message_delay, + default_claim_ttl, + default_claim_grace, + default_subscription_ttl, + max_queues_per_page, + max_messages_per_page, + max_subscriptions_per_page, + max_messages_per_claim_or_pop, + max_queue_metadata, + max_messages_post_size, + max_message_ttl, + max_message_delay, + max_claim_ttl, + max_claim_grace, + subscriber_types, + max_flavors_per_page, + max_pools_per_page +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/storage/mongodb/driver.py b/zaqar/storage/mongodb/driver.py index 793ad244f..2e84c4afa 100644 --- a/zaqar/storage/mongodb/driver.py +++ b/zaqar/storage/mongodb/driver.py @@ -22,10 +22,11 @@ import pymongo import pymongo.errors from zaqar.common import decorators +from zaqar.conf import drivers_management_store_mongodb +from zaqar.conf import drivers_message_store_mongodb from zaqar.i18n import _ from zaqar import storage from zaqar.storage.mongodb import controllers -from zaqar.storage.mongodb import options def _connection(conf): @@ -71,14 +72,17 @@ class DataDriver(storage.DataDriverBase): BASE_CAPABILITIES = tuple(storage.Capabilities) - _DRIVER_OPTIONS = options._config_options() + _DRIVER_OPTIONS = [(drivers_management_store_mongodb.GROUP_NAME, + drivers_management_store_mongodb.ALL_OPTS), + (drivers_message_store_mongodb.GROUP_NAME, + drivers_message_store_mongodb.ALL_OPTS)] _COL_SUFIX = "_messages_p" def __init__(self, conf, cache, control_driver): super(DataDriver, self).__init__(conf, cache, control_driver) - self.mongodb_conf = self.conf[options.MESSAGE_MONGODB_GROUP] + self.mongodb_conf = self.conf[drivers_message_store_mongodb.GROUP_NAME] conn = self.connection server_info = conn.server_info()['version'] @@ -236,10 +240,12 @@ class ControlDriver(storage.ControlDriverBase): def __init__(self, conf, cache): super(ControlDriver, self).__init__(conf, cache) - self.conf.register_opts(options.MANAGEMENT_MONGODB_OPTIONS, - group=options.MANAGEMENT_MONGODB_GROUP) + self.conf.register_opts( + drivers_management_store_mongodb.ALL_OPTS, + group=drivers_management_store_mongodb.GROUP_NAME) - self.mongodb_conf = self.conf[options.MANAGEMENT_MONGODB_GROUP] + self.mongodb_conf = self.conf[ + drivers_management_store_mongodb.GROUP_NAME] def close(self): self.connection.close() diff --git a/zaqar/storage/mongodb/options.py b/zaqar/storage/mongodb/options.py deleted file mode 100644 index 74667601b..000000000 --- a/zaqar/storage/mongodb/options.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright (c) 2013 Rackspace Hosting, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -"""MongoDB storage driver configuration options.""" - -from oslo_config import cfg - -_deprecated_group = 'drivers:storage:mongodb' - -# options that are common to both management and message storage -_COMMON_OPTIONS = ( - - cfg.StrOpt('ssl_keyfile', - deprecated_opts=[cfg.DeprecatedOpt( - 'ssl_keyfile', - group=_deprecated_group), ], - help=('The private keyfile used to identify the local ' - 'connection against mongod. If included with the ' - '``certifle`` then only the ``ssl_certfile`` ' - 'is needed.')), - - cfg.StrOpt('ssl_certfile', - deprecated_opts=[cfg.DeprecatedOpt( - 'ssl_certfile', - group=_deprecated_group), ], - help=('The certificate file used to identify the local ' - 'connection against mongod.')), - - cfg.StrOpt('ssl_cert_reqs', default='CERT_REQUIRED', - deprecated_opts=[cfg.DeprecatedOpt( - 'ssl_cert_reqs', - group=_deprecated_group), ], - help=('Specifies whether a certificate is required from ' - 'the other side of the connection, and whether it ' - 'will be validated if provided. It must be one of ' - 'the three values ``CERT_NONE``(certificates ignored), ' - '``CERT_OPTIONAL``(not required, but validated if ' - 'provided), or ``CERT_REQUIRED``(required and ' - 'validated). If the value of this parameter is not ' - '``CERT_NONE``, then the ``ssl_ca_cert`` parameter ' - 'must point to a file of CA certificates.')), - - cfg.StrOpt('ssl_ca_certs', - deprecated_opts=[cfg.DeprecatedOpt( - 'ssl_ca_certs', - group=_deprecated_group), ], - help=('The ca_certs file contains a set of concatenated ' - '"certification authority" certificates, which are ' - 'used to validate certificates passed from the other ' - 'end of the connection.')), - - cfg.StrOpt('uri', - deprecated_opts=[cfg.DeprecatedOpt( - 'uri', - group=_deprecated_group), ], - help=('Mongodb Connection URI. If ssl connection enabled, ' - 'then ``ssl_keyfile``, ``ssl_certfile``, ' - '``ssl_cert_reqs``, ``ssl_ca_certs`` need to be set ' - 'accordingly.')), - - cfg.StrOpt('database', default='zaqar', - deprecated_opts=[cfg.DeprecatedOpt( - 'database', - group=_deprecated_group), ], - help='Database name.'), - - cfg.IntOpt('max_attempts', min=0, default=1000, - deprecated_opts=[cfg.DeprecatedOpt( - 'max_attempts', - group=_deprecated_group), ], - help=('Maximum number of times to retry a failed operation. ' - 'Currently only used for retrying a message post.')), - - cfg.FloatOpt('max_retry_sleep', default=0.1, - deprecated_opts=[cfg.DeprecatedOpt( - 'max_retry_sleep', - group=_deprecated_group), ], - help=('Maximum sleep interval between retries ' - '(actual sleep time increases linearly ' - 'according to number of attempts performed).')), - - cfg.FloatOpt('max_retry_jitter', default=0.005, - deprecated_opts=[cfg.DeprecatedOpt( - 'max_retry_jitter', - group=_deprecated_group), ], - help=('Maximum jitter interval, to be added to the ' - 'sleep interval, in order to decrease probability ' - 'that parallel requests will retry at the ' - 'same instant.')), - - cfg.IntOpt('max_reconnect_attempts', default=10, - deprecated_opts=[cfg.DeprecatedOpt( - 'max_reconnect_attempts', - group=_deprecated_group), ], - help=('Maximum number of times to retry an operation that ' - 'failed due to a primary node failover.')), - - cfg.FloatOpt('reconnect_sleep', default=0.020, - deprecated_opts=[cfg.DeprecatedOpt( - 'reconnect_sleep', - group=_deprecated_group), ], - help=('Base sleep interval between attempts to reconnect ' - 'after a primary node failover. ' - 'The actual sleep time increases exponentially (power ' - 'of 2) each time the operation is retried.')), -) - -MANAGEMENT_MONGODB_OPTIONS = _COMMON_OPTIONS -MESSAGE_MONGODB_OPTIONS = _COMMON_OPTIONS + ( - # options used only by message_store - cfg.IntOpt('partitions', default=2, - deprecated_opts=[cfg.DeprecatedOpt( - 'partitions', - group=_deprecated_group), ], - help=('Number of databases across which to ' - 'partition message data, in order to ' - 'reduce writer lock %. DO NOT change ' - 'this setting after initial deployment. ' - 'It MUST remain static. Also, you ' - 'should not need a large number of partitions ' - 'to improve performance, esp. if deploying ' - 'MongoDB on SSD storage.')), -) - -MANAGEMENT_MONGODB_GROUP = 'drivers:management_store:mongodb' -MESSAGE_MONGODB_GROUP = 'drivers:message_store:mongodb' - - -def _config_options(): - """Used by config generators. - - Returns a list of (group-name, oslo_config-options) tuples - for management and message storage. - """ - return [(MANAGEMENT_MONGODB_GROUP, MANAGEMENT_MONGODB_OPTIONS), - (MESSAGE_MONGODB_GROUP, MESSAGE_MONGODB_OPTIONS)] diff --git a/zaqar/storage/pipeline.py b/zaqar/storage/pipeline.py index b673e6b3c..f49439608 100644 --- a/zaqar/storage/pipeline.py +++ b/zaqar/storage/pipeline.py @@ -12,7 +12,6 @@ # License for the specific language governing permissions and limitations under # the License. -from oslo_config import cfg from oslo_log import log as logging from osprofiler import profiler from stevedore import driver @@ -20,28 +19,11 @@ from stevedore import extension from zaqar import common from zaqar.common import decorators -from zaqar.i18n import _ +from zaqar.conf import storage from zaqar.storage import base LOG = logging.getLogger(__name__) -_PIPELINE_RESOURCES = ('queue', 'message', 'claim', 'subscription') - -_PIPELINE_CONFIGS = tuple(( - cfg.ListOpt(resource + '_pipeline', default=[], - help=_('Pipeline to use for processing {0} operations. ' - 'This pipeline will be consumed before calling ' - 'the storage driver\'s controller methods.') - .format(resource)) - for resource in _PIPELINE_RESOURCES -)) - -_PIPELINE_GROUP = 'storage' - - -def _config_options(): - return [(_PIPELINE_GROUP, _PIPELINE_CONFIGS)] - def _get_storage_pipeline(resource_name, conf, *args, **kwargs): """Constructs and returns a storage resource pipeline. @@ -64,10 +46,10 @@ def _get_storage_pipeline(resource_name, conf, *args, **kwargs): :returns: A pipeline to use. :rtype: `Pipeline` """ - conf.register_opts(_PIPELINE_CONFIGS, - group=_PIPELINE_GROUP) + conf.register_opts(storage.ALL_OPTS, + group=storage.GROUP_NAME) - storage_conf = conf[_PIPELINE_GROUP] + storage_conf = conf[storage.GROUP_NAME] pipeline = [] for ns in storage_conf[resource_name + '_pipeline']: diff --git a/zaqar/storage/pooling.py b/zaqar/storage/pooling.py index 3dacc59a9..34e10df8a 100644 --- a/zaqar/storage/pooling.py +++ b/zaqar/storage/pooling.py @@ -16,13 +16,13 @@ import heapq import itertools -from oslo_config import cfg from oslo_log import log from osprofiler import profiler from zaqar.common import decorators from zaqar.common import errors as cerrors from zaqar.common.storage import select +from zaqar.conf import pooling_catalog from zaqar.i18n import _ from zaqar import storage from zaqar.storage import errors @@ -31,13 +31,6 @@ from zaqar.storage import utils LOG = log.getLogger(__name__) -_CATALOG_OPTIONS = ( - cfg.BoolOpt('enable_virtual_pool', default=False, - help=('If enabled, the message_store will be used ' - 'as the storage for the virtual pool.')), -) - -_CATALOG_GROUP = 'pooling:catalog' # NOTE(kgriffs): E.g.: 'zaqar-pooling:5083853/my-queue' _POOL_CACHE_PREFIX = 'pooling:' @@ -51,10 +44,6 @@ _POOL_CACHE_PREFIX = 'pooling:' _POOL_CACHE_TTL = 10 -def _config_options(): - return [(_CATALOG_GROUP, _CATALOG_OPTIONS)] - - def _pool_cache_key(queue, project=None): # NOTE(kgriffs): Use string concatenation for performance, # also put project first since it is guaranteed to be @@ -452,8 +441,9 @@ class Catalog(object): self._cache = cache self.control = control - self._conf.register_opts(_CATALOG_OPTIONS, group=_CATALOG_GROUP) - self._catalog_conf = self._conf[_CATALOG_GROUP] + self._conf.register_opts(pooling_catalog.ALL_OPTS, + group=pooling_catalog.GROUP_NAME) + self._catalog_conf = self._conf[pooling_catalog.GROUP_NAME] self._pools_ctrl = control.pools_controller self._flavor_ctrl = control.flavors_controller diff --git a/zaqar/storage/redis/driver.py b/zaqar/storage/redis/driver.py index f77609c3e..ef891ac51 100644 --- a/zaqar/storage/redis/driver.py +++ b/zaqar/storage/redis/driver.py @@ -19,10 +19,11 @@ from six.moves import urllib from zaqar.common import decorators from zaqar.common import errors +from zaqar.conf import drivers_management_store_redis +from zaqar.conf import drivers_message_store_redis from zaqar.i18n import _ from zaqar import storage from zaqar.storage.redis import controllers -from zaqar.storage.redis import options REDIS_DEFAULT_PORT = 6379 SENTINEL_DEFAULT_PORT = 26379 @@ -156,11 +157,14 @@ class DataDriver(storage.DataDriverBase): storage.Capabilities.AOD, storage.Capabilities.HIGH_THROUGHPUT) - _DRIVER_OPTIONS = options._config_options() + _DRIVER_OPTIONS = [(drivers_management_store_redis.GROUP_NAME, + drivers_management_store_redis.ALL_OPTS), + (drivers_message_store_redis.GROUP_NAME, + drivers_message_store_redis.ALL_OPTS)] def __init__(self, conf, cache, control_driver): super(DataDriver, self).__init__(conf, cache, control_driver) - self.redis_conf = self.conf[options.MESSAGE_REDIS_GROUP] + self.redis_conf = self.conf[drivers_message_store_redis.GROUP_NAME] server_version = self.connection.info()['redis_version'] if tuple(map(int, server_version.split('.'))) < (2, 6): @@ -240,10 +244,11 @@ class ControlDriver(storage.ControlDriverBase): def __init__(self, conf, cache): super(ControlDriver, self).__init__(conf, cache) - self.conf.register_opts(options.MANAGEMENT_REDIS_OPTIONS, - group=options.MANAGEMENT_REDIS_GROUP) + self.conf.register_opts( + drivers_management_store_redis.ALL_OPTS, + group=drivers_management_store_redis.GROUP_NAME) - self.redis_conf = self.conf[options.MANAGEMENT_REDIS_GROUP] + self.redis_conf = self.conf[drivers_management_store_redis.GROUP_NAME] def close(self): self.connection.close() diff --git a/zaqar/storage/redis/options.py b/zaqar/storage/redis/options.py deleted file mode 100644 index 1e9cbd306..000000000 --- a/zaqar/storage/redis/options.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) 2014 Prashanth Raghu. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Redis storage driver configuration options.""" - -from oslo_config import cfg - -_deprecated_group = 'drivers:storage:redis' -# options common to management and message storage -_COMMON_REDIS_OPTIONS = ( - cfg.StrOpt('uri', default="redis://127.0.0.1:6379", - deprecated_opts=[cfg.DeprecatedOpt( - 'uri', - group=_deprecated_group), ], - help=('Redis connection URI, taking one of three forms. ' - 'For a direct connection to a Redis server, use ' - 'the form "redis://[:password]@host[:port][?options]", ' - 'where password is redis-server\'s password, when' - 'redis-server is set password, the password option' - 'needs to be set. port defaults to 6379 if not' - 'specified. For an HA master-slave Redis cluster using' - ' Redis Sentinel, use the form ' - '"redis://[:password]@host1[:port1]' - '[,host2[:port2],...,hostN[:portN]][?options]", ' - 'where each host specified corresponds to an ' - 'instance of redis-sentinel. In this form, the ' - 'name of the Redis master used in the Sentinel ' - 'configuration must be included in the query ' - 'string as "master=". Finally, to connect ' - 'to a local instance of Redis over a unix socket, ' - 'you may use the form ' - '"redis:[:password]@/path/to/redis.sock[?options]".' - ' In all forms, the "socket_timeout" option may be' - 'specified in the query string. Its value is ' - 'given in seconds. If not provided, ' - '"socket_timeout" defaults to 0.1 seconds.' - 'There are multiple database instances in redis ' - 'database, for example in the /etc/redis/redis.conf, ' - 'if the parameter is "database 16", there are 16 ' - 'database instances. By default, the data is stored ' - 'in db = 0 database, if you want to use db = 1 ' - 'database, you can use the following form: ' - '"redis://host[:port][?dbid=1]".')), - - cfg.IntOpt('max_reconnect_attempts', default=10, - deprecated_opts=[cfg.DeprecatedOpt( - 'max_reconnect_attempts', - group=_deprecated_group), ], - help=('Maximum number of times to retry an operation that ' - 'failed due to a redis node failover.')), - - cfg.FloatOpt('reconnect_sleep', default=1.0, - deprecated_opts=[cfg.DeprecatedOpt( - 'reconnect_sleep', - group=_deprecated_group), ], - help=('Base sleep interval between attempts to reconnect ' - 'after a redis node failover. ')) - -) - -MANAGEMENT_REDIS_OPTIONS = _COMMON_REDIS_OPTIONS -MESSAGE_REDIS_OPTIONS = _COMMON_REDIS_OPTIONS - -MANAGEMENT_REDIS_GROUP = 'drivers:management_store:redis' -MESSAGE_REDIS_GROUP = 'drivers:message_store:redis' - - -def _config_options(): - return [(MANAGEMENT_REDIS_GROUP, MANAGEMENT_REDIS_OPTIONS), - (MESSAGE_REDIS_GROUP, MESSAGE_REDIS_OPTIONS)] diff --git a/zaqar/storage/sqlalchemy/driver.py b/zaqar/storage/sqlalchemy/driver.py index 394734e80..966d63540 100644 --- a/zaqar/storage/sqlalchemy/driver.py +++ b/zaqar/storage/sqlalchemy/driver.py @@ -19,18 +19,20 @@ from osprofiler import sqlalchemy as sa_tracer import sqlalchemy as sa from zaqar.common import decorators +from zaqar.conf import drivers_management_store_sqlalchemy from zaqar import storage from zaqar.storage.sqlalchemy import controllers -from zaqar.storage.sqlalchemy import options class ControlDriver(storage.ControlDriverBase): def __init__(self, conf, cache): super(ControlDriver, self).__init__(conf, cache) - self.conf.register_opts(options.MANAGEMENT_SQLALCHEMY_OPTIONS, - group=options.MANAGEMENT_SQLALCHEMY_GROUP) - self.sqlalchemy_conf = self.conf[options.MANAGEMENT_SQLALCHEMY_GROUP] + self.conf.register_opts( + drivers_management_store_sqlalchemy.ALL_OPTS, + group=drivers_management_store_sqlalchemy.GROUP_NAME) + self.sqlalchemy_conf = self.conf[ + drivers_management_store_sqlalchemy.GROUP_NAME] def _mysql_on_connect(self, conn, record): # NOTE(flaper87): This is necessary in order diff --git a/zaqar/storage/sqlalchemy/options.py b/zaqar/storage/sqlalchemy/options.py deleted file mode 100644 index 55603d922..000000000 --- a/zaqar/storage/sqlalchemy/options.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2014 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -"""SQLAlchemy storage driver implementation.""" - -from oslo_config import cfg - -_deprecated_group = 'drivers:storage:sqlalchemy' -_COMMON_SQLALCHEMY_OPTIONS = ( - cfg.StrOpt('uri', default='sqlite:///:memory:', - deprecated_opts=[cfg.DeprecatedOpt( - 'uri', - group=_deprecated_group), ], - help='An sqlalchemy URL'), -) - -MANAGEMENT_SQLALCHEMY_OPTIONS = _COMMON_SQLALCHEMY_OPTIONS - -MANAGEMENT_SQLALCHEMY_GROUP = 'drivers:management_store:sqlalchemy' - - -def _config_options(): - return [(MANAGEMENT_SQLALCHEMY_GROUP, MANAGEMENT_SQLALCHEMY_OPTIONS)] diff --git a/zaqar/storage/swift/driver.py b/zaqar/storage/swift/driver.py index b4128df2e..730154c55 100644 --- a/zaqar/storage/swift/driver.py +++ b/zaqar/storage/swift/driver.py @@ -21,20 +21,21 @@ from oslo_log import log as oslo_logging import swiftclient from zaqar.common import decorators +from zaqar.conf import drivers_message_store_swift from zaqar import storage from zaqar.storage.swift import controllers -from zaqar.storage.swift import options LOG = oslo_logging.getLogger(__name__) class DataDriver(storage.DataDriverBase): - _DRIVER_OPTIONS = options._config_options() + _DRIVER_OPTIONS = [(drivers_message_store_swift.GROUP_NAME, + drivers_message_store_swift.ALL_OPTS)] def __init__(self, conf, cache, control_driver): super(DataDriver, self).__init__(conf, cache, control_driver) - self.swift_conf = self.conf[options.MESSAGE_SWIFT_GROUP] + self.swift_conf = self.conf[drivers_message_store_swift.GROUP_NAME] if not self.conf.debug: # Reduce swiftclient logging, in particular to remove 404s logging.getLogger("swiftclient").setLevel(logging.WARNING) diff --git a/zaqar/storage/swift/options.py b/zaqar/storage/swift/options.py deleted file mode 100644 index eddb166c3..000000000 --- a/zaqar/storage/swift/options.py +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Swift storage driver configuration options.""" - -from oslo_config import cfg -MESSAGE_SWIFT_OPTIONS = ( - cfg.StrOpt("auth_url", default="http://127.0.0.1:5000/v3/", - help="URI of Keystone endpoint to discover Swift"), - cfg.StrOpt("uri", - default="swift://demo:nomoresecrete@/demo", - help="Custom URI describing the swift connection."), - cfg.StrOpt("insecure", default=False, help="Don't check SSL certificate"), - cfg.StrOpt("project_domain_id", default="default", - help="Domain ID containing project"), - cfg.StrOpt("project_domain_name", help="Domain name containing project"), - cfg.StrOpt("user_domain_id", default="default", help="User's domain id"), - cfg.StrOpt("user_domain_name", help="User's domain name"), - cfg.StrOpt("region_name", help="Region name"), - cfg.StrOpt("interface", default="publicURL", - help="The default interface for endpoint URL discovery."), -) - - -MESSAGE_SWIFT_GROUP = 'drivers:message_store:swift' - - -def _config_options(): - return [(MESSAGE_SWIFT_GROUP, MESSAGE_SWIFT_OPTIONS), ] diff --git a/zaqar/storage/utils.py b/zaqar/storage/utils.py index fbe0b5589..9130a7400 100644 --- a/zaqar/storage/utils.py +++ b/zaqar/storage/utils.py @@ -128,9 +128,9 @@ def load_storage_driver(conf, cache, storage_type=None, driver_type = 'zaqar.{0}.storage'.format(mode) - _invoke_args = [conf, cache] + _invoke_args = (conf, cache) if control_driver is not None: - _invoke_args.append(control_driver) + _invoke_args = (conf, cache, control_driver) try: mgr = driver.DriverManager(driver_type, diff --git a/zaqar/tests/base.py b/zaqar/tests/base.py index 3b27d3c7a..1e6180578 100644 --- a/zaqar/tests/base.py +++ b/zaqar/tests/base.py @@ -21,7 +21,11 @@ from oslo_log import log from osprofiler import opts import testtools -from zaqar.common import configs +from zaqar.conf import default +from zaqar.conf import drivers +from zaqar.conf import notification +from zaqar.conf import profiler +from zaqar.conf import signed_url from zaqar.tests import helpers @@ -54,18 +58,16 @@ class TestBase(testtools.TestCase): else: self.conf = cfg.ConfigOpts() - self.conf.register_opts(configs._GENERAL_OPTIONS) - self.conf.register_opts(configs._DRIVER_OPTIONS, - group=configs._DRIVER_GROUP) - self.conf.register_opts(configs._NOTIFICATION_OPTIONS, - group=configs._NOTIFICATION_GROUP) - self.conf.register_opts(configs._NOTIFICATION_OPTIONS, - group=configs._NOTIFICATION_GROUP) - self.conf.register_opts(configs._SIGNED_URL_OPTIONS, - group=configs._SIGNED_URL_GROUP) + self.conf.register_opts(default.ALL_OPTS) + self.conf.register_opts(drivers.ALL_OPTS, + group=drivers.GROUP_NAME) + self.conf.register_opts(notification.ALL_OPTS, + group=notification.GROUP_NAME) + self.conf.register_opts(signed_url.ALL_OPTS, + group=signed_url.GROUP_NAME) opts.set_defaults(self.conf) - self.conf.register_opts(configs._PROFILER_OPTIONS, - group=configs._PROFILER_GROUP) + self.conf.register_opts(profiler.ALL_OPTS, + group=profiler.GROUP_NAME) self.redis_url = os.environ.get('ZAQAR_TEST_REDIS_URL', 'redis://127.0.0.1:6379') self.mongodb_url = os.environ.get('ZAQAR_TEST_MONGODB_URL', diff --git a/zaqar/tests/functional/base.py b/zaqar/tests/functional/base.py index a9fc4305c..23f3b6068 100644 --- a/zaqar/tests/functional/base.py +++ b/zaqar/tests/functional/base.py @@ -79,8 +79,6 @@ class FunctionalTestBase(testing.TestBase): validator = validation.Validator(self.mconf) self.limits = validator._limits_conf - transport_base._config_options() - self.resource_defaults = transport_base.ResourceDefaults(self.mconf) # Always register options diff --git a/zaqar/tests/unit/common/storage/test_utils.py b/zaqar/tests/unit/common/storage/test_utils.py index ae6e4f03b..ff31ab628 100644 --- a/zaqar/tests/unit/common/storage/test_utils.py +++ b/zaqar/tests/unit/common/storage/test_utils.py @@ -15,7 +15,7 @@ import ddt -from zaqar.common import configs +from zaqar.conf import default from zaqar.storage import utils from zaqar import tests as testing @@ -25,7 +25,7 @@ class TestUtils(testing.TestBase): def setUp(self): super(TestUtils, self).setUp() - self.conf.register_opts(configs._GENERAL_OPTIONS) + self.conf.register_opts(default.ALL_OPTS) @testing.requires_mongodb def test_can_connect_succeeds_if_good_uri_mongo(self): diff --git a/zaqar/tests/unit/common/test_decorators.py b/zaqar/tests/unit/common/test_decorators.py index 90b2f0cff..d0615f532 100644 --- a/zaqar/tests/unit/common/test_decorators.py +++ b/zaqar/tests/unit/common/test_decorators.py @@ -18,8 +18,8 @@ from oslo_cache import core from oslo_config import cfg from zaqar.common import cache as oslo_cache -from zaqar.common import configs from zaqar.common import decorators +from zaqar.conf import default from zaqar.tests import base @@ -27,7 +27,7 @@ class TestDecorators(base.TestBase): def setUp(self): super(TestDecorators, self).setUp() - self.conf.register_opts(configs._GENERAL_OPTIONS) + self.conf.register_opts(default.ALL_OPTS) def test_memoized_getattr(self): diff --git a/zaqar/tests/unit/storage/test_impl_mongodb.py b/zaqar/tests/unit/storage/test_impl_mongodb.py index 6aa87984d..e5ba2c7cf 100644 --- a/zaqar/tests/unit/storage/test_impl_mongodb.py +++ b/zaqar/tests/unit/storage/test_impl_mongodb.py @@ -26,12 +26,13 @@ import six from testtools import matchers from zaqar.common import cache as oslo_cache -from zaqar.common import configs +from zaqar.conf import default +from zaqar.conf import drivers_management_store_mongodb +from zaqar.conf import drivers_message_store_mongodb from zaqar import storage from zaqar.storage import errors from zaqar.storage import mongodb from zaqar.storage.mongodb import controllers -from zaqar.storage.mongodb import options from zaqar.storage.mongodb import utils from zaqar.storage import pooling from zaqar import tests as testing @@ -51,12 +52,12 @@ class MongodbSetupMixin(object): self.driver.connection.drop_database(db) def _prepare_conf(self): - if options.MESSAGE_MONGODB_GROUP in self.conf: - self.config(options.MESSAGE_MONGODB_GROUP, + if drivers_message_store_mongodb.GROUP_NAME in self.conf: + self.config(drivers_message_store_mongodb.GROUP_NAME, database=uuid.uuid4().hex) - if options.MANAGEMENT_MONGODB_GROUP in self.conf: - self.config(options.MANAGEMENT_MONGODB_GROUP, + if drivers_management_store_mongodb.GROUP_NAME in self.conf: + self.config(drivers_management_store_mongodb.GROUP_NAME, database=uuid.uuid4().hex) @@ -67,10 +68,10 @@ class MongodbUtilsTest(MongodbSetupMixin, testing.TestBase): def setUp(self): super(MongodbUtilsTest, self).setUp() - self.conf.register_opts(options.MESSAGE_MONGODB_OPTIONS, - group=options.MESSAGE_MONGODB_GROUP) + self.conf.register_opts(drivers_message_store_mongodb.ALL_OPTS, + group=drivers_message_store_mongodb.GROUP_NAME) - self.mongodb_conf = self.conf[options.MESSAGE_MONGODB_GROUP] + self.mongodb_conf = self.conf[drivers_message_store_mongodb.GROUP_NAME] MockDriver = collections.namedtuple('MockDriver', 'mongodb_conf') @@ -152,7 +153,7 @@ class MongodbDriverTest(MongodbSetupMixin, testing.TestBase): def setUp(self): super(MongodbDriverTest, self).setUp() - self.conf.register_opts(configs._GENERAL_OPTIONS) + self.conf.register_opts(default.ALL_OPTS) self.config(unreliable=False) oslo_cache.register_config(self.conf) diff --git a/zaqar/tests/unit/storage/test_impl_redis.py b/zaqar/tests/unit/storage/test_impl_redis.py index 2ce1d073d..1837a0d6e 100644 --- a/zaqar/tests/unit/storage/test_impl_redis.py +++ b/zaqar/tests/unit/storage/test_impl_redis.py @@ -23,12 +23,12 @@ import redis from zaqar.common import cache as oslo_cache from zaqar.common import errors +from zaqar.conf import drivers_message_store_redis from zaqar import storage from zaqar.storage import pooling from zaqar.storage.redis import controllers from zaqar.storage.redis import driver from zaqar.storage.redis import messages -from zaqar.storage.redis import options from zaqar.storage.redis import utils from zaqar import tests as testing from zaqar.tests.unit.storage import base @@ -65,10 +65,10 @@ class RedisUtilsTest(testing.TestBase): def setUp(self): super(RedisUtilsTest, self).setUp() - self.conf.register_opts(options.MESSAGE_REDIS_OPTIONS, - group=options.MESSAGE_REDIS_GROUP) + self.conf.register_opts(drivers_message_store_redis.ALL_OPTS, + group=drivers_message_store_redis.GROUP_NAME) - self.redis_conf = self.conf[options.MESSAGE_REDIS_GROUP] + self.redis_conf = self.conf[drivers_message_store_redis.GROUP_NAME] MockDriver = collections.namedtuple('MockDriver', 'redis_conf') diff --git a/zaqar/tests/unit/transport/websocket/base.py b/zaqar/tests/unit/transport/websocket/base.py index 545391061..6c1f0c35f 100644 --- a/zaqar/tests/unit/transport/websocket/base.py +++ b/zaqar/tests/unit/transport/websocket/base.py @@ -15,10 +15,10 @@ from oslo_serialization import jsonutils from zaqar import bootstrap -from zaqar.common import configs +from zaqar.conf import default +from zaqar.conf import drivers_transport_websocket +from zaqar.conf import transport from zaqar import tests as testing -from zaqar.transport import validation -from zaqar.transport.websocket import driver class TestBase(testing.TestBase): @@ -31,14 +31,14 @@ class TestBase(testing.TestBase): if not self.config_file: self.skipTest("No config specified") - self.conf.register_opts(configs._GENERAL_OPTIONS) - self.conf.register_opts(validation._TRANSPORT_LIMITS_OPTIONS, - group=validation._TRANSPORT_LIMITS_GROUP) - self.transport_cfg = self.conf[validation._TRANSPORT_LIMITS_GROUP] + self.conf.register_opts(default.ALL_OPTS) + self.conf.register_opts(transport.ALL_OPTS, + group=transport.GROUP_NAME) + self.transport_cfg = self.conf[transport.GROUP_NAME] - self.conf.register_opts(driver._WS_OPTIONS, - group=driver._WS_GROUP) - self.ws_cfg = self.conf[driver._WS_GROUP] + self.conf.register_opts(drivers_transport_websocket.ALL_OPTS, + group=drivers_transport_websocket.GROUP_NAME) + self.ws_cfg = self.conf[drivers_transport_websocket.GROUP_NAME] self.conf.unreliable = True self.conf.admin_mode = True diff --git a/zaqar/tests/unit/transport/wsgi/base.py b/zaqar/tests/unit/transport/wsgi/base.py index 2b7dcc91a..df0a94124 100644 --- a/zaqar/tests/unit/transport/wsgi/base.py +++ b/zaqar/tests/unit/transport/wsgi/base.py @@ -20,10 +20,10 @@ from oslo_serialization import jsonutils from oslo_utils import uuidutils from zaqar import bootstrap -from zaqar.common import configs +from zaqar.conf import default +from zaqar.conf import drivers_transport_wsgi +from zaqar.conf import transport from zaqar import tests as testing -from zaqar.transport import validation -from zaqar.transport.wsgi import driver class TestBase(testing.TestBase): @@ -36,14 +36,14 @@ class TestBase(testing.TestBase): if not self.config_file: self.skipTest("No config specified") - self.conf.register_opts(configs._GENERAL_OPTIONS) - self.conf.register_opts(validation._TRANSPORT_LIMITS_OPTIONS, - group=validation._TRANSPORT_LIMITS_GROUP) - self.transport_cfg = self.conf[validation._TRANSPORT_LIMITS_GROUP] + self.conf.register_opts(default.ALL_OPTS) + self.conf.register_opts(transport.ALL_OPTS, + group=transport.GROUP_NAME) + self.transport_cfg = self.conf[transport.GROUP_NAME] - self.conf.register_opts(driver._WSGI_OPTIONS, - group=driver._WSGI_GROUP) - self.wsgi_cfg = self.conf[driver._WSGI_GROUP] + self.conf.register_opts(drivers_transport_wsgi.ALL_OPTS, + group=drivers_transport_wsgi.GROUP_NAME) + self.wsgi_cfg = self.conf[drivers_transport_wsgi.GROUP_NAME] self.conf.unreliable = True self.conf.admin_mode = True diff --git a/zaqar/transport/base.py b/zaqar/transport/base.py index 7662853fb..c1ef6384a 100644 --- a/zaqar/transport/base.py +++ b/zaqar/transport/base.py @@ -15,40 +15,10 @@ import abc -from oslo_config import cfg import six - -_GENERAL_TRANSPORT_OPTIONS = ( - cfg.StrOpt('auth_strategy', default='', - help=('Backend to use for authentication. ' - 'For no auth, keep it empty. ' - 'Existing strategies: keystone. ' - 'See also the keystone_authtoken section below')), -) - -_RESOURCE_DEFAULTS = ( - cfg.IntOpt('default_message_ttl', default=3600, - help=('Defines how long a message will be accessible.')), - cfg.IntOpt('default_message_delay', default=0, - help=('Defines the defautl value for queue delay seconds.' - 'The 0 means the delayed queues feature is close.')), - cfg.IntOpt('default_claim_ttl', default=300, - help=('Defines how long a message will be in claimed state.')), - cfg.IntOpt('default_claim_grace', default=60, - help=('Defines the message grace period in seconds.')), - cfg.IntOpt('default_subscription_ttl', default=3600, - help=('Defines how long a subscription will be available.')), -) - -_TRANSPORT_GROUP = 'transport' - - -def _config_options(): - return [ - (None, _GENERAL_TRANSPORT_OPTIONS), - (_TRANSPORT_GROUP, _RESOURCE_DEFAULTS), - ] +from zaqar.conf import default +from zaqar.conf import transport class ResourceDefaults(object): @@ -56,8 +26,9 @@ class ResourceDefaults(object): def __init__(self, conf): self._conf = conf - self._conf.register_opts(_RESOURCE_DEFAULTS, group=_TRANSPORT_GROUP) - self._defaults = self._conf[_TRANSPORT_GROUP] + self._conf.register_opts(transport.ALL_OPTS, + group=transport.GROUP_NAME) + self._defaults = self._conf[transport.GROUP_NAME] @property def message_ttl(self): @@ -96,7 +67,7 @@ class DriverBase(object): self._cache = cache self._control = control - self._conf.register_opts(_GENERAL_TRANSPORT_OPTIONS) + self._conf.register_opts([default.auth_strategy]) self._defaults = ResourceDefaults(self._conf) @abc.abstractmethod diff --git a/zaqar/transport/validation.py b/zaqar/transport/validation.py index f9f8fc215..396dee885 100644 --- a/zaqar/transport/validation.py +++ b/zaqar/transport/validation.py @@ -17,11 +17,11 @@ import datetime import re -from oslo_config import cfg from oslo_utils import timeutils import six from zaqar.common import consts +from zaqar.conf import transport from zaqar.i18n import _ @@ -31,70 +31,6 @@ MIN_CLAIM_GRACE = 60 MIN_DELAY_TTL = 0 MIN_SUBSCRIPTION_TTL = 60 _PURGBLE_RESOURCE_TYPES = {'messages', 'subscriptions'} - -_TRANSPORT_LIMITS_OPTIONS = ( - cfg.IntOpt('max_queues_per_page', default=20, - deprecated_name='queue_paging_uplimit', - deprecated_group='limits:transport', - help='Defines the maximum number of queues per page.'), - - cfg.IntOpt('max_messages_per_page', default=20, - deprecated_name='message_paging_uplimit', - deprecated_group='limits:transport', - help='Defines the maximum number of messages per page.'), - - cfg.IntOpt('max_subscriptions_per_page', default=20, - deprecated_name='subscription_paging_uplimit', - deprecated_group='limits:transport', - help='Defines the maximum number of subscriptions per page.'), - - cfg.IntOpt('max_messages_per_claim_or_pop', default=20, - deprecated_name='max_messages_per_claim', - help='The maximum number of messages that can be claimed (OR) ' - 'popped in a single request'), - - cfg.IntOpt('max_queue_metadata', default=64 * 1024, - deprecated_name='metadata_size_uplimit', - deprecated_group='limits:transport', - help='Defines the maximum amount of metadata in a queue.'), - - cfg.IntOpt('max_messages_post_size', default=256 * 1024, - deprecated_name='message_size_uplimit', - deprecated_group='limits:transport', - deprecated_opts=[cfg.DeprecatedOpt('max_message_size')], - help='Defines the maximum size of message posts.'), - - cfg.IntOpt('max_message_ttl', default=1209600, - deprecated_name='message_ttl_max', - deprecated_group='limits:transport', - help='Maximum amount of time a message will be available.'), - - cfg.IntOpt('max_message_delay', default=900, - help='Maximum delay seconds for messages can be claimed.'), - - cfg.IntOpt('max_claim_ttl', default=43200, - deprecated_name='claim_ttl_max', - deprecated_group='limits:transport', - help='Maximum length of a message in claimed state.'), - - cfg.IntOpt('max_claim_grace', default=43200, - deprecated_name='claim_grace_max', - deprecated_group='limits:transport', - help='Defines the maximum message grace period in seconds.'), - - cfg.ListOpt('subscriber_types', default=['http', 'https', 'mailto', - 'trust+http', 'trust+https'], - help='Defines supported subscriber types.'), - - cfg.IntOpt('max_flavors_per_page', default=20, - help='Defines the maximum number of flavors per page.'), - - cfg.IntOpt('max_pools_per_page', default=20, - help='Defines the maximum number of pools per page.'), -) - -_TRANSPORT_LIMITS_GROUP = 'transport' - # NOTE(kgriffs): Don't use \w because it isn't guaranteed to match # only ASCII characters. QUEUE_NAME_REGEX = re.compile('^[a-zA-Z0-9_\-.]+$') @@ -102,10 +38,6 @@ QUEUE_NAME_MAX_LEN = 64 PROJECT_ID_MAX_LEN = 256 -def _config_options(): - return [(_TRANSPORT_LIMITS_GROUP, _TRANSPORT_LIMITS_OPTIONS)] - - class ValidationFailed(ValueError): """User input did not follow API restrictions.""" @@ -117,9 +49,9 @@ class ValidationFailed(ValueError): class Validator(object): def __init__(self, conf): self._conf = conf - self._conf.register_opts(_TRANSPORT_LIMITS_OPTIONS, - group=_TRANSPORT_LIMITS_GROUP) - self._limits_conf = self._conf[_TRANSPORT_LIMITS_GROUP] + self._conf.register_opts(transport.ALL_OPTS, + group=transport.GROUP_NAME) + self._limits_conf = self._conf[transport.GROUP_NAME] self._supported_operations = ('add', 'remove', 'replace') def queue_identification(self, queue, project): diff --git a/zaqar/transport/websocket/driver.py b/zaqar/transport/websocket/driver.py index 77359a4e1..98595ea67 100644 --- a/zaqar/transport/websocket/driver.py +++ b/zaqar/transport/websocket/driver.py @@ -15,7 +15,6 @@ import socket -from oslo_config import cfg from oslo_log import log as logging from oslo_utils import netutils @@ -25,41 +24,16 @@ except ImportError: import trollius as asyncio from zaqar.common import decorators +from zaqar.conf import drivers_transport_websocket from zaqar.i18n import _ from zaqar.transport import base from zaqar.transport.middleware import auth from zaqar.transport.websocket import factory -_WS_OPTIONS = ( - cfg.HostAddressOpt('bind', default='127.0.0.1', - help='Address on which the self-hosting server will ' - 'listen.'), - - cfg.PortOpt('port', default=9000, - help='Port on which the self-hosting server will listen.'), - - cfg.PortOpt('external-port', - help='Port on which the service is provided to the user.'), - - cfg.HostAddressOpt('notification-bind', - help='Address on which the notification server will ' - 'listen.'), - - cfg.PortOpt('notification-port', default=0, - help='Port on which the notification server will listen.'), - -) - -_WS_GROUP = 'drivers:transport:websocket' - LOG = logging.getLogger(__name__) -def _config_options(): - return [(_WS_GROUP, _WS_OPTIONS)] - - # TODO(derekh): use escape_ipv6 from oslo.utils once available def _escape_ipv6(address): """Escape an IP address in square brackets if IPv6""" @@ -75,8 +49,9 @@ class Driver(base.DriverBase): self._api = api self._cache = cache - self._conf.register_opts(_WS_OPTIONS, group=_WS_GROUP) - self._ws_conf = self._conf[_WS_GROUP] + self._conf.register_opts(drivers_transport_websocket.ALL_OPTS, + group=drivers_transport_websocket.GROUP_NAME) + self._ws_conf = self._conf[drivers_transport_websocket.GROUP_NAME] if self._conf.auth_strategy: auth_strategy = auth.strategy(self._conf.auth_strategy) diff --git a/zaqar/transport/wsgi/driver.py b/zaqar/transport/wsgi/driver.py index 4541a8416..b93da0eb9 100644 --- a/zaqar/transport/wsgi/driver.py +++ b/zaqar/transport/wsgi/driver.py @@ -19,12 +19,12 @@ import six import socket from wsgiref import simple_server -from oslo_config import cfg from oslo_log import log as logging from oslo_utils import netutils from zaqar.common import decorators from zaqar.common.transport.wsgi import helpers +from zaqar.conf import drivers_transport_wsgi from zaqar.i18n import _ from zaqar import transport from zaqar.transport import acl @@ -37,24 +37,10 @@ from zaqar.transport.wsgi import v1_1 from zaqar.transport.wsgi import v2_0 from zaqar.transport.wsgi import version -_WSGI_OPTIONS = ( - cfg.HostAddressOpt('bind', default='127.0.0.1', - help='Address on which the self-hosting server will ' - 'listen.'), - - cfg.PortOpt('port', default=8888, - help='Port on which the self-hosting server will listen.'), -) - -_WSGI_GROUP = 'drivers:transport:wsgi' LOG = logging.getLogger(__name__) -def _config_options(): - return [(_WSGI_GROUP, _WSGI_OPTIONS)] - - class FuncMiddleware(object): def __init__(self, func): @@ -69,8 +55,9 @@ class Driver(transport.DriverBase): def __init__(self, conf, storage, cache, control): super(Driver, self).__init__(conf, storage, cache, control) - self._conf.register_opts(_WSGI_OPTIONS, group=_WSGI_GROUP) - self._wsgi_conf = self._conf[_WSGI_GROUP] + self._conf.register_opts(drivers_transport_wsgi.ALL_OPTS, + group=drivers_transport_wsgi.GROUP_NAME) + self._wsgi_conf = self._conf[drivers_transport_wsgi.GROUP_NAME] self._validate = validation.Validator(self._conf) self.app = None