diff --git a/doc/source/conf.py b/doc/source/conf.py index 6aa28c4a4..cbb9b56d4 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -48,10 +48,9 @@ extensions = ['sphinx.ext.coverage', 'openstackdocstheme', ] -config_generator_config_file = [ - ('../../etc/oslo-config-generator/zaqar.conf', - '_static/zaqar'), -] +config_generator_config_file = '../../etc/oslo-config-generator/zaqar.conf' +sample_config_basename = '_static/zaqar' + # autodoc generation is a bit aggressive and a nuisance # when doing heavy text edit cycles. Execute "export SPHINX_DEBUG=1" diff --git a/doc/source/configuration/zaqar.rst b/doc/source/configuration/zaqar.rst index c080b7f5e..29db5b569 100644 --- a/doc/source/configuration/zaqar.rst +++ b/doc/source/configuration/zaqar.rst @@ -6,4 +6,3 @@ zaqar.conf .. show-options:: :config-file: etc/oslo-config-generator/zaqar.conf - diff --git a/doc/source/install/install-obs.rst b/doc/source/install/install-obs.rst index 176520ff2..1a9d35402 100644 --- a/doc/source/install/install-obs.rst +++ b/doc/source/install/install-obs.rst @@ -278,7 +278,7 @@ Install and configure ``memcached``, ``uWSGI`` and Messaging on the web server .. code-block:: console - # oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf + # oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf # cp etc/zaqar.policy.yaml.sample /etc/zaqar/policy.yaml Edit any item as needed in policy.yaml. diff --git a/doc/source/install/install-rdo.rst b/doc/source/install/install-rdo.rst index 0f49a2f32..27a9ebbdf 100644 --- a/doc/source/install/install-rdo.rst +++ b/doc/source/install/install-rdo.rst @@ -278,7 +278,7 @@ Install and configure ``memcached``, ``uWSGI`` and Messaging on the web server .. code-block:: console - # oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf + # oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf # cp etc/zaqar.policy.yaml.sample /etc/zaqar/policy.yaml Edit any item as needed in policy.yaml. diff --git a/doc/source/install/install-ubuntu.rst b/doc/source/install/install-ubuntu.rst index 2ec14b271..0704026d0 100644 --- a/doc/source/install/install-ubuntu.rst +++ b/doc/source/install/install-ubuntu.rst @@ -266,7 +266,7 @@ Install and configure ``memcached``, ``uWSGI`` and Messaging on the web server .. code-block:: console - # oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf + # oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf # cp etc/zaqar.policy.yaml.sample /etc/zaqar/policy.yaml Edit any item as needed in policy.yaml. diff --git a/etc/README-policy.json.sample b/etc/README-policy.json.sample index 96bb1edc1..a8e0788ab 100644 --- a/etc/README-policy.json.sample +++ b/etc/README-policy.json.sample @@ -3,6 +3,6 @@ level of the zaqar directory: tox -egenpolicy Or run the command directly: - oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf + oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf The output file will be in etc folder, named "zaqar.policy.yaml.sample" by default. \ No newline at end of file diff --git a/etc/oslo-config-generator/zaqar.conf b/etc/oslo-config-generator/zaqar.conf index 0123ea44c..6b41d0073 100644 --- a/etc/oslo-config-generator/zaqar.conf +++ b/etc/oslo-config-generator/zaqar.conf @@ -1,18 +1,7 @@ [DEFAULT] -wrap_width = 80 +wrap_width = 79 output_file = etc/zaqar.conf.sample -namespace = zaqar.common.auth -namespace = zaqar.common.configs -namespace = zaqar.storage.pipeline -namespace = zaqar.storage.pooling -namespace = zaqar.storage.mongodb -namespace = zaqar.storage.redis -namespace = zaqar.storage.sqlalchemy -namespace = zaqar.storage.swift -namespace = zaqar.transport.wsgi -namespace = zaqar.transport.websocket -namespace = zaqar.transport.base -namespace = zaqar.transport.validation +namespace = zaqar namespace = keystonemiddleware.auth_token namespace = oslo.cache namespace = oslo.log diff --git a/etc/oslo-config-generator/zaqar-policy-generator.conf b/etc/zaqar-policy-generator.conf similarity index 100% rename from etc/oslo-config-generator/zaqar-policy-generator.conf rename to etc/zaqar-policy-generator.conf diff --git a/releasenotes/notes/configuration-refactor-0ff219ac59c96347.yaml b/releasenotes/notes/configuration-refactor-0ff219ac59c96347.yaml new file mode 100644 index 000000000..ee821b4d0 --- /dev/null +++ b/releasenotes/notes/configuration-refactor-0ff219ac59c96347.yaml @@ -0,0 +1,6 @@ +other: + - | + The code structure for configuration files are changed. This is insensitvie + for end users, but the persons who work for downstream changes should pay + attention. Please refactor your private configurations to ``zaqar/conf/`` + folder as well. diff --git a/setup.cfg b/setup.cfg index dbb4c45a6..59a9f496e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,18 +46,7 @@ zaqar.transport = websocket = zaqar.transport.websocket.driver:Driver oslo.config.opts = - zaqar.common.auth = zaqar.common.auth:_config_options - zaqar.common.configs = zaqar.common.configs:_config_options - zaqar.storage.pipeline = zaqar.storage.pipeline:_config_options - zaqar.storage.pooling = zaqar.storage.pooling:_config_options - zaqar.storage.mongodb = zaqar.storage.mongodb.options:_config_options - zaqar.storage.redis = zaqar.storage.redis.options:_config_options - zaqar.storage.sqlalchemy = zaqar.storage.sqlalchemy.options:_config_options - zaqar.storage.swift = zaqar.storage.swift.options:_config_options - zaqar.transport.wsgi = zaqar.transport.wsgi.driver:_config_options - zaqar.transport.websocket = zaqar.transport.websocket.driver:_config_options - zaqar.transport.base = zaqar.transport.base:_config_options - zaqar.transport.validation = zaqar.transport.validation:_config_options + zaqar = zaqar.conf.opts:list_opts zaqar.storage.stages = zaqar.notification.notifier = zaqar.notification.notifier:NotifierDriver diff --git a/tox.ini b/tox.ini index b57dddf35..22fa5c907 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,7 @@ commands = [testenv:genpolicy] commands = - oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf + oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf [testenv:cover] commands = diff --git a/zaqar/bootstrap.py b/zaqar/bootstrap.py index 91d47dcc2..bdf97cd5e 100644 --- a/zaqar/bootstrap.py +++ b/zaqar/bootstrap.py @@ -22,10 +22,11 @@ from stevedore import driver from zaqar.api import handler from zaqar.common import cache as oslo_cache -from zaqar.common import configs from zaqar.common import consts from zaqar.common import decorators from zaqar.common import errors +from zaqar.conf import drivers as driver_opts +from zaqar.conf import opts as opts_tool from zaqar.storage import pipeline from zaqar.storage import pooling from zaqar.storage import utils as storage_utils @@ -46,7 +47,7 @@ class Bootstrap(object): def __init__(self, conf): self.conf = conf - for group, opts in configs._config_options(): + for group, opts in opts_tool.list_opts_by_group(): self.conf.register_opts(opts, group=group) profiler_opts.set_defaults(self.conf) @@ -54,7 +55,7 @@ class Bootstrap(object): # wsgi. Websocket part will be added in the future. profile.setup(self.conf, 'Zaqar-server', socket.gethostname()) - self.driver_conf = self.conf[configs._DRIVER_GROUP] + self.driver_conf = self.conf[driver_opts.GROUP_NAME] @decorators.lazy_property(write=False) def api(self): diff --git a/zaqar/cmd/server.py b/zaqar/cmd/server.py index ae1673f34..55e04a89a 100644 --- a/zaqar/cmd/server.py +++ b/zaqar/cmd/server.py @@ -21,12 +21,12 @@ from oslo_reports import opts as gmr_opts from zaqar import bootstrap from zaqar.common import cli -from zaqar.common import configs +from zaqar.conf import default from zaqar import version # NOTE(eggmaster): define command line options for zaqar-server _CLI_OPTIONS = ( - configs._ADMIN_MODE_OPT, + default.admin_mode, cfg.BoolOpt('daemon', default=False, help='Run Zaqar server in the background.'), ) diff --git a/zaqar/common/configs.py b/zaqar/common/configs.py deleted file mode 100644 index 916e9a33e..000000000 --- a/zaqar/common/configs.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright (c) 2015 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from oslo_config import cfg - - -_ADMIN_MODE_OPT = cfg.BoolOpt('admin_mode', default=False, - help='Activate privileged endpoints.') - - -_GENERAL_OPTIONS = ( - _ADMIN_MODE_OPT, - cfg.BoolOpt('pooling', default=False, - help=('Enable pooling across multiple storage backends. ' - 'If pooling is enabled, the storage driver ' - 'configuration is used to determine where the ' - 'catalogue/control plane data is kept.'), - deprecated_opts=[cfg.DeprecatedOpt('sharding')]), - cfg.BoolOpt('unreliable', default=False, - help='Disable all reliability constraints.'), - cfg.ListOpt('enable_deprecated_api_versions', default=[], - item_type=cfg.types.List(item_type=cfg.types.String( - choices=('1', '1.1'))), - help='List of deprecated API versions to enable.'), - cfg.BoolOpt('enable_checksum', default=False, - help='Enable a checksum for message body. The default value ' - 'is False.'), -) - -_DRIVER_OPTIONS = ( - cfg.StrOpt('transport', default='wsgi', - help='Transport driver to use.'), - cfg.StrOpt('message_store', default='mongodb', - deprecated_opts=[cfg.DeprecatedOpt('storage')], - help='Storage driver to use as the messaging store.'), - cfg.StrOpt('management_store', default='mongodb', - help='Storage driver to use as the management store.'), -) - -_DRIVER_GROUP = 'drivers' - - -_SIGNED_URL_OPTIONS = ( - cfg.StrOpt('secret_key', - help=('Secret key used to encrypt pre-signed URLs.')), -) - -_SIGNED_URL_GROUP = 'signed_url' - - -_NOTIFICATION_OPTIONS = ( - cfg.StrOpt('smtp_command', default='/usr/sbin/sendmail -t -oi', - help=('The command of smtp to send email. The format is ' - '"command_name arg1 arg2".')), - cfg.IntOpt('max_notifier_workers', default=10, - help='The max amount of the notification workers.'), - cfg.BoolOpt('require_confirmation', default=False, - help='Whether the http/https/email subscription need to be ' - 'confirmed before notification.'), - cfg.StrOpt('external_confirmation_url', - help='The confirmation page url that will be used in email ' - 'subscription confirmation before notification.'), - cfg.DictOpt("subscription_confirmation_email_template", - default={'topic': 'Zaqar Notification - Subscription ' - 'Confirmation', - 'body': 'You have chosen to subscribe to the ' - 'queue: {0}. This queue belongs to ' - 'project: {1}. ' - 'To confirm this subscription, ' - 'click or visit this link below: {2}', - 'sender': 'Zaqar Notifications ' - ''}, - help="Defines the set of subscription confirmation email " - "content, including topic, body and sender. There is " - "a mapping is {0} -> queue name, {1} ->project id, " - "{2}-> confirm url in body string. User can use any of " - "the three value. But they can't use more than three."), - cfg.DictOpt("unsubscribe_confirmation_email_template", - default={'topic': 'Zaqar Notification - ' - 'Unsubscribe Confirmation', - 'body': 'You have unsubscribed successfully to the ' - 'queue: {0}. This queue belongs to ' - 'project: {1}. ' - 'To resubscribe this subscription, ' - 'click or visit this link below: {2}', - 'sender': 'Zaqar Notifications ' - ''}, - help="Defines the set of unsubscribe confirmation email " - "content, including topic, body and sender. There is " - "a mapping is {0} -> queue name, {1} ->project id, " - "{2}-> confirm url in body string. User can use any of " - "the three value. But they can't use more than three."), -) - -_NOTIFICATION_GROUP = 'notification' - - -_PROFILER_OPTIONS = [ - cfg.BoolOpt("trace_wsgi_transport", default=False, - help="If False doesn't trace any transport requests." - "Please note that it doesn't work for websocket now."), - cfg.BoolOpt("trace_message_store", default=False, - help="If False doesn't trace any message store requests."), - cfg.BoolOpt("trace_management_store", default=False, - help="If False doesn't trace any management store requests.") -] - -_PROFILER_GROUP = "profiler" - - -def _config_options(): - return [(None, _GENERAL_OPTIONS), - (_DRIVER_GROUP, _DRIVER_OPTIONS), - (_SIGNED_URL_GROUP, _SIGNED_URL_OPTIONS), - (_NOTIFICATION_GROUP, _NOTIFICATION_OPTIONS), - (_PROFILER_GROUP, _PROFILER_OPTIONS)] diff --git a/zaqar/conf/__init__.py b/zaqar/conf/__init__.py new file mode 100644 index 000000000..d565c4be0 --- /dev/null +++ b/zaqar/conf/__init__.py @@ -0,0 +1,74 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from oslo_cache import core as cache +from oslo_config import cfg +from oslo_log import log + +from zaqar.conf import default +from zaqar.conf import drivers +from zaqar.conf import drivers_management_store_mongodb +from zaqar.conf import drivers_management_store_redis +from zaqar.conf import drivers_management_store_sqlalchemy +from zaqar.conf import drivers_message_store_mongodb +from zaqar.conf import drivers_message_store_redis +from zaqar.conf import drivers_message_store_swift +from zaqar.conf import drivers_transport_websocket +from zaqar.conf import drivers_transport_wsgi +from zaqar.conf import notification +from zaqar.conf import pooling_catalog +from zaqar.conf import profiler +from zaqar.conf import signed_url +from zaqar.conf import storage +from zaqar.conf import transport + +CONF = cfg.CONF + + +conf_modules = [ + default, + drivers, + drivers_management_store_mongodb, + drivers_management_store_redis, + drivers_management_store_sqlalchemy, + drivers_message_store_mongodb, + drivers_message_store_redis, + drivers_message_store_swift, + drivers_transport_websocket, + drivers_transport_wsgi, + notification, + pooling_catalog, + profiler, + signed_url, + storage, + transport +] + + +def setup_logging(): + """Set up logging for the keystone package.""" + log.setup(CONF, 'zaqar') + logging.captureWarnings(True) + + +def configure(conf=None): + if conf is None: + conf = CONF + + for module in conf_modules: + module.register_opts(conf) + + # add oslo.cache related config options + cache.configure(conf) diff --git a/zaqar/conf/default.py b/zaqar/conf/default.py new file mode 100644 index 000000000..15c31903f --- /dev/null +++ b/zaqar/conf/default.py @@ -0,0 +1,70 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +admin_mode = cfg.BoolOpt( + 'admin_mode', default=False, + help='Activate privileged endpoints.') + + +pooling = cfg.BoolOpt( + 'pooling', default=False, + help=('Enable pooling across multiple storage backends. ' + 'If pooling is enabled, the storage driver ' + 'configuration is used to determine where the ' + 'catalogue/control plane data is kept.'), + deprecated_opts=[cfg.DeprecatedOpt('sharding')]) + + +unreliable = cfg.BoolOpt( + 'unreliable', default=False, + help='Disable all reliability constraints.') + + +enable_deprecated_api_versions = cfg.ListOpt( + 'enable_deprecated_api_versions', default=[], + item_type=cfg.types.List(item_type=cfg.types.String(choices=('1', '1.1'))), + help='List of deprecated API versions to enable.') + + +enable_checksum = cfg.BoolOpt( + 'enable_checksum', default=False, + help='Enable a checksum for message body. The default value is False.') + + +auth_strategy = cfg.StrOpt( + 'auth_strategy', default='', + help=('Backend to use for authentication. ' + 'For no auth, keep it empty. ' + 'Existing strategies: keystone. ' + 'See also the keystone_authtoken section below')) + +GROUP_NAME = 'DEFAULT' +ALL_OPTS = [ + admin_mode, + pooling, + unreliable, + enable_deprecated_api_versions, + enable_checksum, + auth_strategy +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers.py b/zaqar/conf/drivers.py new file mode 100644 index 000000000..3110ef243 --- /dev/null +++ b/zaqar/conf/drivers.py @@ -0,0 +1,46 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +transport = cfg.StrOpt( + 'transport', default='wsgi', + help='Transport driver to use.') + + +message_store = cfg.StrOpt( + 'message_store', default='mongodb', + deprecated_opts=[cfg.DeprecatedOpt('storage')], + help='Storage driver to use as the messaging store.') + + +management_store = cfg.StrOpt( + 'management_store', default='mongodb', + help='Storage driver to use as the management store.') + + +GROUP_NAME = 'drivers' +ALL_OPTS = [ + transport, + message_store, + management_store +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_management_store_mongodb.py b/zaqar/conf/drivers_management_store_mongodb.py new file mode 100644 index 000000000..782fe7789 --- /dev/null +++ b/zaqar/conf/drivers_management_store_mongodb.py @@ -0,0 +1,157 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +_deprecated_group = 'drivers:storage:mongodb' + + +ssl_keyfile = cfg.StrOpt( + 'ssl_keyfile', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_keyfile', + group=_deprecated_group), ], + help='The private keyfile used to identify the local ' + 'connection against mongod. If included with ' + 'the ``certifle`` then only the ``ssl_certfile``' + ' is needed.') + + +ssl_certfile = cfg.StrOpt( + 'ssl_certfile', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_certfile', + group=_deprecated_group), ], + help='The certificate file used to identify the ' + 'local connection against mongod.') + + +ssl_cert_reqs = cfg.StrOpt( + 'ssl_cert_reqs', default='CERT_REQUIRED', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_cert_reqs', + group=_deprecated_group), ], + help='Specifies whether a certificate is required ' + 'from the other side of the connection, and ' + 'whether it will be validated if provided. It ' + 'must be one of the three values ``CERT_NONE``' + '(certificates ignored), ``CERT_OPTIONAL``' + '(not required, but validated if provided), or' + ' ``CERT_REQUIRED``(required and validated). ' + 'If the value of this parameter is not ' + '``CERT_NONE``, then the ``ssl_ca_cert`` ' + 'parameter must point to a file of CA ' + 'certificates.') + + +ssl_ca_certs = cfg.StrOpt( + 'ssl_ca_certs', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_ca_certs', + group=_deprecated_group), ], + help='The ca_certs file contains a set of concatenated ' + '"certification authority" certificates, which are ' + 'used to validate certificates passed from the other ' + 'end of the connection.') + + +uri = cfg.StrOpt( + 'uri', + deprecated_opts=[cfg.DeprecatedOpt( + 'uri', + group=_deprecated_group), ], + help='Mongodb Connection URI. If ssl connection enabled, ' + 'then ``ssl_keyfile``, ``ssl_certfile``, ' + '``ssl_cert_reqs``, ``ssl_ca_certs`` need to be set ' + 'accordingly.') + + +database = cfg.StrOpt( + 'database', default='zaqar', + deprecated_opts=[cfg.DeprecatedOpt( + 'database', + group=_deprecated_group), ], + help='Database name.') + +max_attempts = cfg.IntOpt( + 'max_attempts', min=0, default=1000, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_attempts', + group=_deprecated_group), ], + help=('Maximum number of times to retry a failed operation. ' + 'Currently only used for retrying a message post.')) + + +max_retry_sleep = cfg.FloatOpt( + 'max_retry_sleep', default=0.1, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_retry_sleep', + group=_deprecated_group), ], + help=('Maximum sleep interval between retries ' + '(actual sleep time increases linearly ' + 'according to number of attempts performed).')) + + +max_retry_jitter = cfg.FloatOpt( + 'max_retry_jitter', default=0.005, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_retry_jitter', + group=_deprecated_group), ], + help=('Maximum jitter interval, to be added to the ' + 'sleep interval, in order to decrease probability ' + 'that parallel requests will retry at the ' + 'same instant.')) + + +max_reconnect_attempts = cfg.IntOpt( + 'max_reconnect_attempts', default=10, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_reconnect_attempts', + group=_deprecated_group), ], + help=('Maximum number of times to retry an operation that ' + 'failed due to a primary node failover.')) + + +reconnect_sleep = cfg.FloatOpt( + 'reconnect_sleep', default=0.020, + deprecated_opts=[cfg.DeprecatedOpt( + 'reconnect_sleep', + group=_deprecated_group), ], + help=('Base sleep interval between attempts to reconnect ' + 'after a primary node failover. ' + 'The actual sleep time increases exponentially (power ' + 'of 2) each time the operation is retried.')) + + +GROUP_NAME = 'drivers:management_store:mongodb' +ALL_OPTS = [ + ssl_keyfile, + ssl_certfile, + ssl_cert_reqs, + ssl_ca_certs, + uri, + database, + max_attempts, + max_retry_sleep, + max_retry_jitter, + max_reconnect_attempts, + reconnect_sleep +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_management_store_redis.py b/zaqar/conf/drivers_management_store_redis.py new file mode 100644 index 000000000..d6be0c6cf --- /dev/null +++ b/zaqar/conf/drivers_management_store_redis.py @@ -0,0 +1,86 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +_deprecated_group = 'drivers:storage:redis' + + +uri = cfg.StrOpt( + 'uri', default="redis://127.0.0.1:6379", + deprecated_opts=[cfg.DeprecatedOpt( + 'uri', + group=_deprecated_group), ], + help=('Redis connection URI, taking one of three forms. ' + 'For a direct connection to a Redis server, use ' + 'the form "redis://[:password]@host[:port][?options]", ' + 'where password is redis-server\'s password, when' + 'redis-server is set password, the password option' + 'needs to be set. port defaults to 6379 if not' + 'specified. For an HA master-slave Redis cluster using' + ' Redis Sentinel, use the form ' + '"redis://[:password]@host1[:port1]' + '[,host2[:port2],...,hostN[:portN]][?options]", ' + 'where each host specified corresponds to an ' + 'instance of redis-sentinel. In this form, the ' + 'name of the Redis master used in the Sentinel ' + 'configuration must be included in the query ' + 'string as "master=". Finally, to connect ' + 'to a local instance of Redis over a unix socket, ' + 'you may use the form ' + '"redis:[:password]@/path/to/redis.sock[?options]".' + ' In all forms, the "socket_timeout" option may be' + 'specified in the query string. Its value is ' + 'given in seconds. If not provided, ' + '"socket_timeout" defaults to 0.1 seconds.' + 'There are multiple database instances in redis ' + 'database, for example in the /etc/redis/redis.conf, ' + 'if the parameter is "database 16", there are 16 ' + 'database instances. By default, the data is stored ' + 'in db = 0 database, if you want to use db = 1 ' + 'database, you can use the following form: ' + '"redis://host[:port][?dbid=1]".')) + + +max_reconnect_attempts = cfg.IntOpt( + 'max_reconnect_attempts', default=10, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_reconnect_attempts', + group=_deprecated_group), ], + help=('Maximum number of times to retry an operation that ' + 'failed due to a redis node failover.')) + + +reconnect_sleep = cfg.FloatOpt( + 'reconnect_sleep', default=1.0, + deprecated_opts=[cfg.DeprecatedOpt( + 'reconnect_sleep', + group=_deprecated_group), ], + help=('Base sleep interval between attempts to reconnect ' + 'after a redis node failover. ')) + + +GROUP_NAME = 'drivers:management_store:redis' +ALL_OPTS = [ + uri, + max_reconnect_attempts, + reconnect_sleep +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_management_store_sqlalchemy.py b/zaqar/conf/drivers_management_store_sqlalchemy.py new file mode 100644 index 000000000..3de3feb25 --- /dev/null +++ b/zaqar/conf/drivers_management_store_sqlalchemy.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +_deprecated_group = 'drivers:storage:sqlalchemy' + +uri = cfg.StrOpt( + 'uri', default='sqlite:///:memory:', + deprecated_opts=[cfg.DeprecatedOpt( + 'uri', + group=_deprecated_group), ], + help='An sqlalchemy URL') + + +GROUP_NAME = 'drivers:management_store:sqlalchemy' +ALL_OPTS = [ + uri +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_message_store_mongodb.py b/zaqar/conf/drivers_message_store_mongodb.py new file mode 100644 index 000000000..314023a35 --- /dev/null +++ b/zaqar/conf/drivers_message_store_mongodb.py @@ -0,0 +1,173 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +_deprecated_group = 'drivers:storage:mongodb' + + +ssl_keyfile = cfg.StrOpt( + 'ssl_keyfile', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_keyfile', + group=_deprecated_group), ], + help='The private keyfile used to identify the local ' + 'connection against mongod. If included with ' + 'the ``certifle`` then only the ``ssl_certfile``' + ' is needed.') + + +ssl_certfile = cfg.StrOpt( + 'ssl_certfile', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_certfile', + group=_deprecated_group), ], + help='The certificate file used to identify the ' + 'local connection against mongod.') + + +ssl_cert_reqs = cfg.StrOpt( + 'ssl_cert_reqs', default='CERT_REQUIRED', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_cert_reqs', + group=_deprecated_group), ], + help='Specifies whether a certificate is required ' + 'from the other side of the connection, and ' + 'whether it will be validated if provided. It ' + 'must be one of the three values ``CERT_NONE``' + '(certificates ignored), ``CERT_OPTIONAL``' + '(not required, but validated if provided), or' + ' ``CERT_REQUIRED``(required and validated). ' + 'If the value of this parameter is not ' + '``CERT_NONE``, then the ``ssl_ca_cert`` ' + 'parameter must point to a file of CA ' + 'certificates.') + + +ssl_ca_certs = cfg.StrOpt( + 'ssl_ca_certs', + deprecated_opts=[cfg.DeprecatedOpt( + 'ssl_ca_certs', + group=_deprecated_group), ], + help='The ca_certs file contains a set of concatenated ' + '"certification authority" certificates, which are ' + 'used to validate certificates passed from the other ' + 'end of the connection.') + + +uri = cfg.StrOpt( + 'uri', + deprecated_opts=[cfg.DeprecatedOpt( + 'uri', + group=_deprecated_group), ], + help='Mongodb Connection URI. If ssl connection enabled, ' + 'then ``ssl_keyfile``, ``ssl_certfile``, ' + '``ssl_cert_reqs``, ``ssl_ca_certs`` need to be set ' + 'accordingly.') + + +database = cfg.StrOpt( + 'database', default='zaqar', + deprecated_opts=[cfg.DeprecatedOpt( + 'database', + group=_deprecated_group), ], + help='Database name.') + +max_attempts = cfg.IntOpt( + 'max_attempts', min=0, default=1000, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_attempts', + group=_deprecated_group), ], + help=('Maximum number of times to retry a failed operation. ' + 'Currently only used for retrying a message post.')) + + +max_retry_sleep = cfg.FloatOpt( + 'max_retry_sleep', default=0.1, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_retry_sleep', + group=_deprecated_group), ], + help=('Maximum sleep interval between retries ' + '(actual sleep time increases linearly ' + 'according to number of attempts performed).')) + + +max_retry_jitter = cfg.FloatOpt( + 'max_retry_jitter', default=0.005, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_retry_jitter', + group=_deprecated_group), ], + help=('Maximum jitter interval, to be added to the ' + 'sleep interval, in order to decrease probability ' + 'that parallel requests will retry at the ' + 'same instant.')) + + +max_reconnect_attempts = cfg.IntOpt( + 'max_reconnect_attempts', default=10, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_reconnect_attempts', + group=_deprecated_group), ], + help=('Maximum number of times to retry an operation that ' + 'failed due to a primary node failover.')) + + +reconnect_sleep = cfg.FloatOpt( + 'reconnect_sleep', default=0.020, + deprecated_opts=[cfg.DeprecatedOpt( + 'reconnect_sleep', + group=_deprecated_group), ], + help=('Base sleep interval between attempts to reconnect ' + 'after a primary node failover. ' + 'The actual sleep time increases exponentially (power ' + 'of 2) each time the operation is retried.')) + + +partitions = cfg.IntOpt( + 'partitions', default=2, + deprecated_opts=[cfg.DeprecatedOpt( + 'partitions', + group=_deprecated_group), ], + help=('Number of databases across which to ' + 'partition message data, in order to ' + 'reduce writer lock %. DO NOT change ' + 'this setting after initial deployment. ' + 'It MUST remain static. Also, you ' + 'should not need a large number of partitions ' + 'to improve performance, esp. if deploying ' + 'MongoDB on SSD storage.')) + + +GROUP_NAME = 'drivers:message_store:mongodb' +ALL_OPTS = [ + ssl_keyfile, + ssl_certfile, + ssl_cert_reqs, + ssl_ca_certs, + uri, + database, + max_attempts, + max_retry_sleep, + max_retry_jitter, + max_reconnect_attempts, + reconnect_sleep, + partitions +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_message_store_redis.py b/zaqar/conf/drivers_message_store_redis.py new file mode 100644 index 000000000..1584d8e18 --- /dev/null +++ b/zaqar/conf/drivers_message_store_redis.py @@ -0,0 +1,86 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +_deprecated_group = 'drivers:storage:redis' + + +uri = cfg.StrOpt( + 'uri', default="redis://127.0.0.1:6379", + deprecated_opts=[cfg.DeprecatedOpt( + 'uri', + group=_deprecated_group), ], + help=('Redis connection URI, taking one of three forms. ' + 'For a direct connection to a Redis server, use ' + 'the form "redis://[:password]@host[:port][?options]", ' + 'where password is redis-server\'s password, when' + 'redis-server is set password, the password option' + 'needs to be set. port defaults to 6379 if not' + 'specified. For an HA master-slave Redis cluster using' + ' Redis Sentinel, use the form ' + '"redis://[:password]@host1[:port1]' + '[,host2[:port2],...,hostN[:portN]][?options]", ' + 'where each host specified corresponds to an ' + 'instance of redis-sentinel. In this form, the ' + 'name of the Redis master used in the Sentinel ' + 'configuration must be included in the query ' + 'string as "master=". Finally, to connect ' + 'to a local instance of Redis over a unix socket, ' + 'you may use the form ' + '"redis:[:password]@/path/to/redis.sock[?options]".' + ' In all forms, the "socket_timeout" option may be' + 'specified in the query string. Its value is ' + 'given in seconds. If not provided, ' + '"socket_timeout" defaults to 0.1 seconds.' + 'There are multiple database instances in redis ' + 'database, for example in the /etc/redis/redis.conf, ' + 'if the parameter is "database 16", there are 16 ' + 'database instances. By default, the data is stored ' + 'in db = 0 database, if you want to use db = 1 ' + 'database, you can use the following form: ' + '"redis://host[:port][?dbid=1]".')) + + +max_reconnect_attempts = cfg.IntOpt( + 'max_reconnect_attempts', default=10, + deprecated_opts=[cfg.DeprecatedOpt( + 'max_reconnect_attempts', + group=_deprecated_group), ], + help=('Maximum number of times to retry an operation that ' + 'failed due to a redis node failover.')) + + +reconnect_sleep = cfg.FloatOpt( + 'reconnect_sleep', default=1.0, + deprecated_opts=[cfg.DeprecatedOpt( + 'reconnect_sleep', + group=_deprecated_group), ], + help=('Base sleep interval between attempts to reconnect ' + 'after a redis node failover. ')) + + +GROUP_NAME = 'drivers:message_store:redis' +ALL_OPTS = [ + uri, + max_reconnect_attempts, + reconnect_sleep +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_message_store_swift.py b/zaqar/conf/drivers_message_store_swift.py new file mode 100644 index 000000000..f9e2d0a3a --- /dev/null +++ b/zaqar/conf/drivers_message_store_swift.py @@ -0,0 +1,80 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +auth_url = cfg.StrOpt( + "auth_url", default="http://127.0.0.1:5000/v3/", + help="URI of Keystone endpoint to discover Swift") + + +uri = cfg.StrOpt( + "uri", + default="swift://demo:nomoresecrete@/demo", + help="Custom URI describing the swift connection.") + + +insecure = cfg.StrOpt( + "insecure", default=False, + help="Don't check SSL certificate") + + +project_domain_id = cfg.StrOpt( + "project_domain_id", default="default", + help="Domain ID containing project") + + +project_domain_name = cfg.StrOpt( + "project_domain_name", + help="Domain name containing project") + + +user_domain_id = cfg.StrOpt( + "user_domain_id", default="default", + help="User's domain id") + + +user_domain_name = cfg.StrOpt( + "user_domain_name", help="User's domain name") + + +region_name = cfg.StrOpt( + "region_name", help="Region name") + + +interface = cfg.StrOpt( + "interface", default="publicURL", + help="The default interface for endpoint URL " + "discovery.") + + +GROUP_NAME = 'drivers:message_store:swift' +ALL_OPTS = [ + auth_url, + uri, + insecure, + project_domain_id, + project_domain_name, + user_domain_id, + user_domain_name, + region_name, + interface +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_transport_websocket.py b/zaqar/conf/drivers_transport_websocket.py new file mode 100644 index 000000000..ed66c701d --- /dev/null +++ b/zaqar/conf/drivers_transport_websocket.py @@ -0,0 +1,59 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +bind = cfg.HostAddressOpt( + 'bind', default='127.0.0.1', + help='Address on which the self-hosting server will ' + 'listen.') + + +port = cfg.PortOpt( + 'port', default=9000, + help='Port on which the self-hosting server will listen.') + + +external_port = cfg.PortOpt( + 'external-port', + help='Port on which the service is provided to the user.') + + +notification_bind = cfg.HostAddressOpt( + 'notification-bind', + help='Address on which the notification server will ' + 'listen.') + + +notification_port = cfg.PortOpt( + 'notification-port', default=0, + help='Port on which the notification server will listen.') + + +GROUP_NAME = 'drivers:transport:websocket' +ALL_OPTS = [ + bind, + port, + external_port, + notification_bind, + notification_port +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/drivers_transport_wsgi.py b/zaqar/conf/drivers_transport_wsgi.py new file mode 100644 index 000000000..95e13d7bb --- /dev/null +++ b/zaqar/conf/drivers_transport_wsgi.py @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +bind = cfg.HostAddressOpt( + 'bind', default='127.0.0.1', + help='Address on which the self-hosting server will ' + 'listen.') + + +port = cfg.PortOpt( + 'port', default=8888, + help='Port on which the self-hosting server will listen.') + + +GROUP_NAME = 'drivers:transport:wsgi' +ALL_OPTS = [ + bind, + port +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/notification.py b/zaqar/conf/notification.py new file mode 100644 index 000000000..38653ce1b --- /dev/null +++ b/zaqar/conf/notification.py @@ -0,0 +1,93 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +smtp_command = cfg.StrOpt( + 'smtp_command', default='/usr/sbin/sendmail -t -oi', + help=( + 'The command of smtp to send email. The format is ' + '"command_name arg1 arg2".')) + + +max_notifier_workers = cfg.IntOpt( + 'max_notifier_workers', default=10, + help='The max amount of the notification workers.') + + +require_confirmation = cfg.BoolOpt( + 'require_confirmation', default=False, + help='Whether the http/https/email subscription need to be confirmed ' + 'before notification.') + + +external_confirmation_url = cfg.StrOpt( + 'external_confirmation_url', + help='The confirmation page url that will be used in email subscription ' + 'confirmation before notification.') + + +subscription_confirmation_email_template = cfg.DictOpt( + "subscription_confirmation_email_template", + default={'topic': 'Zaqar Notification - Subscription ' + 'Confirmation', + 'body': 'You have chosen to subscribe to the ' + 'queue: {0}. This queue belongs to ' + 'project: {1}. ' + 'To confirm this subscription, ' + 'click or visit this link below: {2}', + 'sender': 'Zaqar Notifications ' + ''}, + help="Defines the set of subscription confirmation email content, " + "including topic, body and sender. There is a mapping is " + "{0} -> queue name, {1} ->project id, {2}-> confirm url in body " + "string. User can use any of the three values. But they can't use " + "more than three.") + + +unsubscribe_confirmation_email_template = cfg.DictOpt( + "unsubscribe_confirmation_email_template", + default={'topic': 'Zaqar Notification - ' + 'Unsubscribe Confirmation', + 'body': 'You have unsubscribed successfully to the ' + 'queue: {0}. This queue belongs to ' + 'project: {1}. ' + 'To resubscribe this subscription, ' + 'click or visit this link below: {2}', + 'sender': 'Zaqar Notifications ' + ''}, + help="Defines the set of unsubscribe confirmation email content, " + "including topic, body and sender. There is a mapping is " + "{0} -> queue name, {1} ->project id, {2}-> confirm url in body " + "string. User can use any of the three values. But they can't use " + "more than three.") + + +GROUP_NAME = 'notification' +ALL_OPTS = [ + smtp_command, + max_notifier_workers, + require_confirmation, + external_confirmation_url, + subscription_confirmation_email_template, + unsubscribe_confirmation_email_template +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/opts.py b/zaqar/conf/opts.py new file mode 100644 index 000000000..e826f4af9 --- /dev/null +++ b/zaqar/conf/opts.py @@ -0,0 +1,109 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Single point of entry to generate the sample configuration file. + +This module collects all the necessary info from the other modules in this +package. It is assumed that: + +* Every other module in this package has a 'list_opts' function which + returns a dict where: + + * The keys are strings which are the group names. + + * The value of each key is a list of config options for that group. + +* The conf package doesn't have further packages with config options. + +* This module is only used in the context of sample file generation. + +""" + +import collections +import importlib +import os +import pkgutil + + +LIST_OPTS_FUNC_NAME = 'list_opts' +IGNORED_MODULES = ('opts', 'constants', 'utils') + + +def list_opts(): + opts = collections.defaultdict(list) + module_names = _list_module_names() + imported_modules = _import_modules(module_names) + _append_config_options(imported_modules, opts) + return _tupleize(opts) + + +def list_opts_by_group(): + opts = [] + module_names = _list_module_names() + imported_modules = _import_modules(module_names) + for module in imported_modules: + configs = module.list_opts() + group_name = (module.GROUP_NAME if module.GROUP_NAME != 'DEFAULT' + else None) + opts.append((group_name, configs[module.GROUP_NAME])) + return opts + + +def _tupleize(d): + """Convert a dict of options to the 2-tuple format.""" + return [(key, value) for key, value in d.items()] + + +def _list_module_names(): + module_names = [] + package_path = os.path.dirname(os.path.abspath(__file__)) + for _, module_name, ispkg in pkgutil.iter_modules(path=[package_path]): + if module_name in IGNORED_MODULES or ispkg: + # Skip this module. + continue + else: + module_names.append(module_name) + return module_names + + +def _import_modules(module_names): + imported_modules = [] + for module_name in module_names: + full_module_path = '.'.join(__name__.split('.')[:-1] + [module_name]) + module = importlib.import_module(full_module_path) + if not hasattr(module, LIST_OPTS_FUNC_NAME): + raise Exception( + "The module '%s' should have a '%s' function which " + "returns the config options." % ( + full_module_path, + LIST_OPTS_FUNC_NAME)) + else: + imported_modules.append(module) + return imported_modules + + +def _process_old_opts(configs): + """Convert old-style 2-tuple configs to dicts.""" + if isinstance(configs, tuple): + configs = [configs] + return {label: options for label, options in configs} + + +def _append_config_options(imported_modules, config_options): + for module in imported_modules: + configs = module.list_opts() + # TODO(markus_z): Remove this compatibility shim once all list_opts() + # functions have been updated to return dicts. + if not isinstance(configs, dict): + configs = _process_old_opts(configs) + for key, val in configs.items(): + config_options[key].extend(val) diff --git a/zaqar/conf/pooling_catalog.py b/zaqar/conf/pooling_catalog.py new file mode 100644 index 000000000..32beff5b0 --- /dev/null +++ b/zaqar/conf/pooling_catalog.py @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +enable_virtual_pool = cfg.BoolOpt( + 'enable_virtual_pool', default=False, + help='If enabled, the message_store will be used as the storage for the ' + 'virtual pool.') + + +GROUP_NAME = 'pooling:catalog' +ALL_OPTS = [ + enable_virtual_pool +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/profiler.py b/zaqar/conf/profiler.py new file mode 100644 index 000000000..87a1124e4 --- /dev/null +++ b/zaqar/conf/profiler.py @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +trace_wsgi_transport = cfg.BoolOpt( + "trace_wsgi_transport", default=False, + help="If False doesn't trace any transport requests." + "Please note that it doesn't work for websocket now.") + + +trace_message_store = cfg.BoolOpt( + "trace_message_store", default=False, + help="If False doesn't trace any message store requests.") + + +trace_management_store = cfg.BoolOpt( + "trace_management_store", default=False, + help="If False doesn't trace any management store requests.") + + +GROUP_NAME = 'profiler' +ALL_OPTS = [ + trace_wsgi_transport, + trace_message_store, + trace_management_store +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/signed_url.py b/zaqar/conf/signed_url.py new file mode 100644 index 000000000..151b8057d --- /dev/null +++ b/zaqar/conf/signed_url.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +secret_key = cfg.StrOpt('secret_key', + help='Secret key used to encrypt pre-signed URLs.') + + +GROUP_NAME = 'signed_url' +ALL_OPTS = [ + secret_key +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/storage.py b/zaqar/conf/storage.py new file mode 100644 index 000000000..2779e3c83 --- /dev/null +++ b/zaqar/conf/storage.py @@ -0,0 +1,61 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +from zaqar.i18n import _ + + +queue_pipeline = cfg.ListOpt( + 'queue_pipeline', default=[], + help=_('Pipeline to use for processing queue operations. This pipeline ' + 'will be consumed before calling the storage driver\'s controller ' + 'methods.')) + + +message_pipeline = cfg.ListOpt( + 'message_pipeline', default=[], + help=_('Pipeline to use for processing message operations. This pipeline ' + 'will be consumed before calling the storage driver\'s controller ' + 'methods.')) + + +claim_pipeline = cfg.ListOpt( + 'claim_pipeline', default=[], + help=_('Pipeline to use for processing claim operations. This pipeline ' + 'will be consumed before calling the storage driver\'s controller ' + 'methods.')) + + +subscription_pipeline = cfg.ListOpt( + 'subscription_pipeline', default=[], + help=_('Pipeline to use for processing subscription operations. This ' + 'pipeline will be consumed before calling the storage driver\'s ' + 'controller methods.')) + + +GROUP_NAME = 'storage' +ALL_OPTS = [ + queue_pipeline, + message_pipeline, + claim_pipeline, + subscription_pipeline +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/conf/transport.py b/zaqar/conf/transport.py new file mode 100644 index 000000000..0a5fd71a9 --- /dev/null +++ b/zaqar/conf/transport.py @@ -0,0 +1,155 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +default_message_ttl = cfg.IntOpt( + 'default_message_ttl', default=3600, + help='Defines how long a message will be accessible.') + + +default_message_delay = cfg.IntOpt( + 'default_message_delay', default=0, + help=('Defines the defautl value for queue delay seconds.' + 'The 0 means the delayed queues feature is close.')) + + +default_claim_ttl = cfg.IntOpt( + 'default_claim_ttl', default=300, + help='Defines how long a message will be in claimed state.') + + +default_claim_grace = cfg.IntOpt( + 'default_claim_grace', default=60, + help='Defines the message grace period in seconds.') + + +default_subscription_ttl = cfg.IntOpt( + 'default_subscription_ttl', default=3600, + help='Defines how long a subscription will be available.') + + +max_queues_per_page = cfg.IntOpt( + 'max_queues_per_page', default=20, + deprecated_name='queue_paging_uplimit', + deprecated_group='limits:transport', + help='Defines the maximum number of queues per page.') + + +max_messages_per_page = cfg.IntOpt( + 'max_messages_per_page', default=20, + deprecated_name='message_paging_uplimit', + deprecated_group='limits:transport', + help='Defines the maximum number of messages per page.') + + +max_subscriptions_per_page = cfg.IntOpt( + 'max_subscriptions_per_page', default=20, + deprecated_name='subscription_paging_uplimit', + deprecated_group='limits:transport', + help='Defines the maximum number of subscriptions per page.') + + +max_messages_per_claim_or_pop = cfg.IntOpt( + 'max_messages_per_claim_or_pop', default=20, + deprecated_name='max_messages_per_claim', + help='The maximum number of messages that can be claimed (OR) ' + 'popped in a single request') + + +max_queue_metadata = cfg.IntOpt( + 'max_queue_metadata', default=64 * 1024, + deprecated_name='metadata_size_uplimit', + deprecated_group='limits:transport', + help='Defines the maximum amount of metadata in a queue.') + + +max_messages_post_size = cfg.IntOpt( + 'max_messages_post_size', default=256 * 1024, + deprecated_name='message_size_uplimit', + deprecated_group='limits:transport', + deprecated_opts=[cfg.DeprecatedOpt('max_message_size')], + help='Defines the maximum size of message posts.') + + +max_message_ttl = cfg.IntOpt( + 'max_message_ttl', default=1209600, + deprecated_name='message_ttl_max', + deprecated_group='limits:transport', + help='Maximum amount of time a message will be available.') + + +max_message_delay = cfg.IntOpt( + 'max_message_delay', default=900, + help='Maximum delay seconds for messages can be claimed.') + + +max_claim_ttl = cfg.IntOpt( + 'max_claim_ttl', default=43200, + deprecated_name='claim_ttl_max', + deprecated_group='limits:transport', + help='Maximum length of a message in claimed state.') + + +max_claim_grace = cfg.IntOpt( + 'max_claim_grace', default=43200, + deprecated_name='claim_grace_max', + deprecated_group='limits:transport', + help='Defines the maximum message grace period in seconds.') + + +subscriber_types = cfg.ListOpt( + 'subscriber_types', default=['http', 'https', 'mailto', + 'trust+http', 'trust+https'], + help='Defines supported subscriber types.') + + +max_flavors_per_page = cfg.IntOpt( + 'max_flavors_per_page', default=20, + help='Defines the maximum number of flavors per page.') + + +max_pools_per_page = cfg.IntOpt( + 'max_pools_per_page', default=20, + help='Defines the maximum number of pools per page.') + + +GROUP_NAME = 'transport' +ALL_OPTS = [ + default_message_ttl, + default_message_delay, + default_claim_ttl, + default_claim_grace, + default_subscription_ttl, + max_queues_per_page, + max_messages_per_page, + max_subscriptions_per_page, + max_messages_per_claim_or_pop, + max_queue_metadata, + max_messages_post_size, + max_message_ttl, + max_message_delay, + max_claim_ttl, + max_claim_grace, + subscriber_types, + max_flavors_per_page, + max_pools_per_page +] + + +def register_opts(conf): + conf.register_opts(ALL_OPTS, group=GROUP_NAME) + + +def list_opts(): + return {GROUP_NAME: ALL_OPTS} diff --git a/zaqar/storage/mongodb/driver.py b/zaqar/storage/mongodb/driver.py index 793ad244f..2e84c4afa 100644 --- a/zaqar/storage/mongodb/driver.py +++ b/zaqar/storage/mongodb/driver.py @@ -22,10 +22,11 @@ import pymongo import pymongo.errors from zaqar.common import decorators +from zaqar.conf import drivers_management_store_mongodb +from zaqar.conf import drivers_message_store_mongodb from zaqar.i18n import _ from zaqar import storage from zaqar.storage.mongodb import controllers -from zaqar.storage.mongodb import options def _connection(conf): @@ -71,14 +72,17 @@ class DataDriver(storage.DataDriverBase): BASE_CAPABILITIES = tuple(storage.Capabilities) - _DRIVER_OPTIONS = options._config_options() + _DRIVER_OPTIONS = [(drivers_management_store_mongodb.GROUP_NAME, + drivers_management_store_mongodb.ALL_OPTS), + (drivers_message_store_mongodb.GROUP_NAME, + drivers_message_store_mongodb.ALL_OPTS)] _COL_SUFIX = "_messages_p" def __init__(self, conf, cache, control_driver): super(DataDriver, self).__init__(conf, cache, control_driver) - self.mongodb_conf = self.conf[options.MESSAGE_MONGODB_GROUP] + self.mongodb_conf = self.conf[drivers_message_store_mongodb.GROUP_NAME] conn = self.connection server_info = conn.server_info()['version'] @@ -236,10 +240,12 @@ class ControlDriver(storage.ControlDriverBase): def __init__(self, conf, cache): super(ControlDriver, self).__init__(conf, cache) - self.conf.register_opts(options.MANAGEMENT_MONGODB_OPTIONS, - group=options.MANAGEMENT_MONGODB_GROUP) + self.conf.register_opts( + drivers_management_store_mongodb.ALL_OPTS, + group=drivers_management_store_mongodb.GROUP_NAME) - self.mongodb_conf = self.conf[options.MANAGEMENT_MONGODB_GROUP] + self.mongodb_conf = self.conf[ + drivers_management_store_mongodb.GROUP_NAME] def close(self): self.connection.close() diff --git a/zaqar/storage/mongodb/options.py b/zaqar/storage/mongodb/options.py deleted file mode 100644 index 74667601b..000000000 --- a/zaqar/storage/mongodb/options.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright (c) 2013 Rackspace Hosting, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -"""MongoDB storage driver configuration options.""" - -from oslo_config import cfg - -_deprecated_group = 'drivers:storage:mongodb' - -# options that are common to both management and message storage -_COMMON_OPTIONS = ( - - cfg.StrOpt('ssl_keyfile', - deprecated_opts=[cfg.DeprecatedOpt( - 'ssl_keyfile', - group=_deprecated_group), ], - help=('The private keyfile used to identify the local ' - 'connection against mongod. If included with the ' - '``certifle`` then only the ``ssl_certfile`` ' - 'is needed.')), - - cfg.StrOpt('ssl_certfile', - deprecated_opts=[cfg.DeprecatedOpt( - 'ssl_certfile', - group=_deprecated_group), ], - help=('The certificate file used to identify the local ' - 'connection against mongod.')), - - cfg.StrOpt('ssl_cert_reqs', default='CERT_REQUIRED', - deprecated_opts=[cfg.DeprecatedOpt( - 'ssl_cert_reqs', - group=_deprecated_group), ], - help=('Specifies whether a certificate is required from ' - 'the other side of the connection, and whether it ' - 'will be validated if provided. It must be one of ' - 'the three values ``CERT_NONE``(certificates ignored), ' - '``CERT_OPTIONAL``(not required, but validated if ' - 'provided), or ``CERT_REQUIRED``(required and ' - 'validated). If the value of this parameter is not ' - '``CERT_NONE``, then the ``ssl_ca_cert`` parameter ' - 'must point to a file of CA certificates.')), - - cfg.StrOpt('ssl_ca_certs', - deprecated_opts=[cfg.DeprecatedOpt( - 'ssl_ca_certs', - group=_deprecated_group), ], - help=('The ca_certs file contains a set of concatenated ' - '"certification authority" certificates, which are ' - 'used to validate certificates passed from the other ' - 'end of the connection.')), - - cfg.StrOpt('uri', - deprecated_opts=[cfg.DeprecatedOpt( - 'uri', - group=_deprecated_group), ], - help=('Mongodb Connection URI. If ssl connection enabled, ' - 'then ``ssl_keyfile``, ``ssl_certfile``, ' - '``ssl_cert_reqs``, ``ssl_ca_certs`` need to be set ' - 'accordingly.')), - - cfg.StrOpt('database', default='zaqar', - deprecated_opts=[cfg.DeprecatedOpt( - 'database', - group=_deprecated_group), ], - help='Database name.'), - - cfg.IntOpt('max_attempts', min=0, default=1000, - deprecated_opts=[cfg.DeprecatedOpt( - 'max_attempts', - group=_deprecated_group), ], - help=('Maximum number of times to retry a failed operation. ' - 'Currently only used for retrying a message post.')), - - cfg.FloatOpt('max_retry_sleep', default=0.1, - deprecated_opts=[cfg.DeprecatedOpt( - 'max_retry_sleep', - group=_deprecated_group), ], - help=('Maximum sleep interval between retries ' - '(actual sleep time increases linearly ' - 'according to number of attempts performed).')), - - cfg.FloatOpt('max_retry_jitter', default=0.005, - deprecated_opts=[cfg.DeprecatedOpt( - 'max_retry_jitter', - group=_deprecated_group), ], - help=('Maximum jitter interval, to be added to the ' - 'sleep interval, in order to decrease probability ' - 'that parallel requests will retry at the ' - 'same instant.')), - - cfg.IntOpt('max_reconnect_attempts', default=10, - deprecated_opts=[cfg.DeprecatedOpt( - 'max_reconnect_attempts', - group=_deprecated_group), ], - help=('Maximum number of times to retry an operation that ' - 'failed due to a primary node failover.')), - - cfg.FloatOpt('reconnect_sleep', default=0.020, - deprecated_opts=[cfg.DeprecatedOpt( - 'reconnect_sleep', - group=_deprecated_group), ], - help=('Base sleep interval between attempts to reconnect ' - 'after a primary node failover. ' - 'The actual sleep time increases exponentially (power ' - 'of 2) each time the operation is retried.')), -) - -MANAGEMENT_MONGODB_OPTIONS = _COMMON_OPTIONS -MESSAGE_MONGODB_OPTIONS = _COMMON_OPTIONS + ( - # options used only by message_store - cfg.IntOpt('partitions', default=2, - deprecated_opts=[cfg.DeprecatedOpt( - 'partitions', - group=_deprecated_group), ], - help=('Number of databases across which to ' - 'partition message data, in order to ' - 'reduce writer lock %. DO NOT change ' - 'this setting after initial deployment. ' - 'It MUST remain static. Also, you ' - 'should not need a large number of partitions ' - 'to improve performance, esp. if deploying ' - 'MongoDB on SSD storage.')), -) - -MANAGEMENT_MONGODB_GROUP = 'drivers:management_store:mongodb' -MESSAGE_MONGODB_GROUP = 'drivers:message_store:mongodb' - - -def _config_options(): - """Used by config generators. - - Returns a list of (group-name, oslo_config-options) tuples - for management and message storage. - """ - return [(MANAGEMENT_MONGODB_GROUP, MANAGEMENT_MONGODB_OPTIONS), - (MESSAGE_MONGODB_GROUP, MESSAGE_MONGODB_OPTIONS)] diff --git a/zaqar/storage/pipeline.py b/zaqar/storage/pipeline.py index b673e6b3c..f49439608 100644 --- a/zaqar/storage/pipeline.py +++ b/zaqar/storage/pipeline.py @@ -12,7 +12,6 @@ # License for the specific language governing permissions and limitations under # the License. -from oslo_config import cfg from oslo_log import log as logging from osprofiler import profiler from stevedore import driver @@ -20,28 +19,11 @@ from stevedore import extension from zaqar import common from zaqar.common import decorators -from zaqar.i18n import _ +from zaqar.conf import storage from zaqar.storage import base LOG = logging.getLogger(__name__) -_PIPELINE_RESOURCES = ('queue', 'message', 'claim', 'subscription') - -_PIPELINE_CONFIGS = tuple(( - cfg.ListOpt(resource + '_pipeline', default=[], - help=_('Pipeline to use for processing {0} operations. ' - 'This pipeline will be consumed before calling ' - 'the storage driver\'s controller methods.') - .format(resource)) - for resource in _PIPELINE_RESOURCES -)) - -_PIPELINE_GROUP = 'storage' - - -def _config_options(): - return [(_PIPELINE_GROUP, _PIPELINE_CONFIGS)] - def _get_storage_pipeline(resource_name, conf, *args, **kwargs): """Constructs and returns a storage resource pipeline. @@ -64,10 +46,10 @@ def _get_storage_pipeline(resource_name, conf, *args, **kwargs): :returns: A pipeline to use. :rtype: `Pipeline` """ - conf.register_opts(_PIPELINE_CONFIGS, - group=_PIPELINE_GROUP) + conf.register_opts(storage.ALL_OPTS, + group=storage.GROUP_NAME) - storage_conf = conf[_PIPELINE_GROUP] + storage_conf = conf[storage.GROUP_NAME] pipeline = [] for ns in storage_conf[resource_name + '_pipeline']: diff --git a/zaqar/storage/pooling.py b/zaqar/storage/pooling.py index 3dacc59a9..34e10df8a 100644 --- a/zaqar/storage/pooling.py +++ b/zaqar/storage/pooling.py @@ -16,13 +16,13 @@ import heapq import itertools -from oslo_config import cfg from oslo_log import log from osprofiler import profiler from zaqar.common import decorators from zaqar.common import errors as cerrors from zaqar.common.storage import select +from zaqar.conf import pooling_catalog from zaqar.i18n import _ from zaqar import storage from zaqar.storage import errors @@ -31,13 +31,6 @@ from zaqar.storage import utils LOG = log.getLogger(__name__) -_CATALOG_OPTIONS = ( - cfg.BoolOpt('enable_virtual_pool', default=False, - help=('If enabled, the message_store will be used ' - 'as the storage for the virtual pool.')), -) - -_CATALOG_GROUP = 'pooling:catalog' # NOTE(kgriffs): E.g.: 'zaqar-pooling:5083853/my-queue' _POOL_CACHE_PREFIX = 'pooling:' @@ -51,10 +44,6 @@ _POOL_CACHE_PREFIX = 'pooling:' _POOL_CACHE_TTL = 10 -def _config_options(): - return [(_CATALOG_GROUP, _CATALOG_OPTIONS)] - - def _pool_cache_key(queue, project=None): # NOTE(kgriffs): Use string concatenation for performance, # also put project first since it is guaranteed to be @@ -452,8 +441,9 @@ class Catalog(object): self._cache = cache self.control = control - self._conf.register_opts(_CATALOG_OPTIONS, group=_CATALOG_GROUP) - self._catalog_conf = self._conf[_CATALOG_GROUP] + self._conf.register_opts(pooling_catalog.ALL_OPTS, + group=pooling_catalog.GROUP_NAME) + self._catalog_conf = self._conf[pooling_catalog.GROUP_NAME] self._pools_ctrl = control.pools_controller self._flavor_ctrl = control.flavors_controller diff --git a/zaqar/storage/redis/driver.py b/zaqar/storage/redis/driver.py index f77609c3e..ef891ac51 100644 --- a/zaqar/storage/redis/driver.py +++ b/zaqar/storage/redis/driver.py @@ -19,10 +19,11 @@ from six.moves import urllib from zaqar.common import decorators from zaqar.common import errors +from zaqar.conf import drivers_management_store_redis +from zaqar.conf import drivers_message_store_redis from zaqar.i18n import _ from zaqar import storage from zaqar.storage.redis import controllers -from zaqar.storage.redis import options REDIS_DEFAULT_PORT = 6379 SENTINEL_DEFAULT_PORT = 26379 @@ -156,11 +157,14 @@ class DataDriver(storage.DataDriverBase): storage.Capabilities.AOD, storage.Capabilities.HIGH_THROUGHPUT) - _DRIVER_OPTIONS = options._config_options() + _DRIVER_OPTIONS = [(drivers_management_store_redis.GROUP_NAME, + drivers_management_store_redis.ALL_OPTS), + (drivers_message_store_redis.GROUP_NAME, + drivers_message_store_redis.ALL_OPTS)] def __init__(self, conf, cache, control_driver): super(DataDriver, self).__init__(conf, cache, control_driver) - self.redis_conf = self.conf[options.MESSAGE_REDIS_GROUP] + self.redis_conf = self.conf[drivers_message_store_redis.GROUP_NAME] server_version = self.connection.info()['redis_version'] if tuple(map(int, server_version.split('.'))) < (2, 6): @@ -240,10 +244,11 @@ class ControlDriver(storage.ControlDriverBase): def __init__(self, conf, cache): super(ControlDriver, self).__init__(conf, cache) - self.conf.register_opts(options.MANAGEMENT_REDIS_OPTIONS, - group=options.MANAGEMENT_REDIS_GROUP) + self.conf.register_opts( + drivers_management_store_redis.ALL_OPTS, + group=drivers_management_store_redis.GROUP_NAME) - self.redis_conf = self.conf[options.MANAGEMENT_REDIS_GROUP] + self.redis_conf = self.conf[drivers_management_store_redis.GROUP_NAME] def close(self): self.connection.close() diff --git a/zaqar/storage/redis/options.py b/zaqar/storage/redis/options.py deleted file mode 100644 index 1e9cbd306..000000000 --- a/zaqar/storage/redis/options.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) 2014 Prashanth Raghu. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Redis storage driver configuration options.""" - -from oslo_config import cfg - -_deprecated_group = 'drivers:storage:redis' -# options common to management and message storage -_COMMON_REDIS_OPTIONS = ( - cfg.StrOpt('uri', default="redis://127.0.0.1:6379", - deprecated_opts=[cfg.DeprecatedOpt( - 'uri', - group=_deprecated_group), ], - help=('Redis connection URI, taking one of three forms. ' - 'For a direct connection to a Redis server, use ' - 'the form "redis://[:password]@host[:port][?options]", ' - 'where password is redis-server\'s password, when' - 'redis-server is set password, the password option' - 'needs to be set. port defaults to 6379 if not' - 'specified. For an HA master-slave Redis cluster using' - ' Redis Sentinel, use the form ' - '"redis://[:password]@host1[:port1]' - '[,host2[:port2],...,hostN[:portN]][?options]", ' - 'where each host specified corresponds to an ' - 'instance of redis-sentinel. In this form, the ' - 'name of the Redis master used in the Sentinel ' - 'configuration must be included in the query ' - 'string as "master=". Finally, to connect ' - 'to a local instance of Redis over a unix socket, ' - 'you may use the form ' - '"redis:[:password]@/path/to/redis.sock[?options]".' - ' In all forms, the "socket_timeout" option may be' - 'specified in the query string. Its value is ' - 'given in seconds. If not provided, ' - '"socket_timeout" defaults to 0.1 seconds.' - 'There are multiple database instances in redis ' - 'database, for example in the /etc/redis/redis.conf, ' - 'if the parameter is "database 16", there are 16 ' - 'database instances. By default, the data is stored ' - 'in db = 0 database, if you want to use db = 1 ' - 'database, you can use the following form: ' - '"redis://host[:port][?dbid=1]".')), - - cfg.IntOpt('max_reconnect_attempts', default=10, - deprecated_opts=[cfg.DeprecatedOpt( - 'max_reconnect_attempts', - group=_deprecated_group), ], - help=('Maximum number of times to retry an operation that ' - 'failed due to a redis node failover.')), - - cfg.FloatOpt('reconnect_sleep', default=1.0, - deprecated_opts=[cfg.DeprecatedOpt( - 'reconnect_sleep', - group=_deprecated_group), ], - help=('Base sleep interval between attempts to reconnect ' - 'after a redis node failover. ')) - -) - -MANAGEMENT_REDIS_OPTIONS = _COMMON_REDIS_OPTIONS -MESSAGE_REDIS_OPTIONS = _COMMON_REDIS_OPTIONS - -MANAGEMENT_REDIS_GROUP = 'drivers:management_store:redis' -MESSAGE_REDIS_GROUP = 'drivers:message_store:redis' - - -def _config_options(): - return [(MANAGEMENT_REDIS_GROUP, MANAGEMENT_REDIS_OPTIONS), - (MESSAGE_REDIS_GROUP, MESSAGE_REDIS_OPTIONS)] diff --git a/zaqar/storage/sqlalchemy/driver.py b/zaqar/storage/sqlalchemy/driver.py index 394734e80..966d63540 100644 --- a/zaqar/storage/sqlalchemy/driver.py +++ b/zaqar/storage/sqlalchemy/driver.py @@ -19,18 +19,20 @@ from osprofiler import sqlalchemy as sa_tracer import sqlalchemy as sa from zaqar.common import decorators +from zaqar.conf import drivers_management_store_sqlalchemy from zaqar import storage from zaqar.storage.sqlalchemy import controllers -from zaqar.storage.sqlalchemy import options class ControlDriver(storage.ControlDriverBase): def __init__(self, conf, cache): super(ControlDriver, self).__init__(conf, cache) - self.conf.register_opts(options.MANAGEMENT_SQLALCHEMY_OPTIONS, - group=options.MANAGEMENT_SQLALCHEMY_GROUP) - self.sqlalchemy_conf = self.conf[options.MANAGEMENT_SQLALCHEMY_GROUP] + self.conf.register_opts( + drivers_management_store_sqlalchemy.ALL_OPTS, + group=drivers_management_store_sqlalchemy.GROUP_NAME) + self.sqlalchemy_conf = self.conf[ + drivers_management_store_sqlalchemy.GROUP_NAME] def _mysql_on_connect(self, conn, record): # NOTE(flaper87): This is necessary in order diff --git a/zaqar/storage/sqlalchemy/options.py b/zaqar/storage/sqlalchemy/options.py deleted file mode 100644 index 55603d922..000000000 --- a/zaqar/storage/sqlalchemy/options.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2014 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -"""SQLAlchemy storage driver implementation.""" - -from oslo_config import cfg - -_deprecated_group = 'drivers:storage:sqlalchemy' -_COMMON_SQLALCHEMY_OPTIONS = ( - cfg.StrOpt('uri', default='sqlite:///:memory:', - deprecated_opts=[cfg.DeprecatedOpt( - 'uri', - group=_deprecated_group), ], - help='An sqlalchemy URL'), -) - -MANAGEMENT_SQLALCHEMY_OPTIONS = _COMMON_SQLALCHEMY_OPTIONS - -MANAGEMENT_SQLALCHEMY_GROUP = 'drivers:management_store:sqlalchemy' - - -def _config_options(): - return [(MANAGEMENT_SQLALCHEMY_GROUP, MANAGEMENT_SQLALCHEMY_OPTIONS)] diff --git a/zaqar/storage/swift/driver.py b/zaqar/storage/swift/driver.py index b4128df2e..730154c55 100644 --- a/zaqar/storage/swift/driver.py +++ b/zaqar/storage/swift/driver.py @@ -21,20 +21,21 @@ from oslo_log import log as oslo_logging import swiftclient from zaqar.common import decorators +from zaqar.conf import drivers_message_store_swift from zaqar import storage from zaqar.storage.swift import controllers -from zaqar.storage.swift import options LOG = oslo_logging.getLogger(__name__) class DataDriver(storage.DataDriverBase): - _DRIVER_OPTIONS = options._config_options() + _DRIVER_OPTIONS = [(drivers_message_store_swift.GROUP_NAME, + drivers_message_store_swift.ALL_OPTS)] def __init__(self, conf, cache, control_driver): super(DataDriver, self).__init__(conf, cache, control_driver) - self.swift_conf = self.conf[options.MESSAGE_SWIFT_GROUP] + self.swift_conf = self.conf[drivers_message_store_swift.GROUP_NAME] if not self.conf.debug: # Reduce swiftclient logging, in particular to remove 404s logging.getLogger("swiftclient").setLevel(logging.WARNING) diff --git a/zaqar/storage/swift/options.py b/zaqar/storage/swift/options.py deleted file mode 100644 index eddb166c3..000000000 --- a/zaqar/storage/swift/options.py +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Swift storage driver configuration options.""" - -from oslo_config import cfg -MESSAGE_SWIFT_OPTIONS = ( - cfg.StrOpt("auth_url", default="http://127.0.0.1:5000/v3/", - help="URI of Keystone endpoint to discover Swift"), - cfg.StrOpt("uri", - default="swift://demo:nomoresecrete@/demo", - help="Custom URI describing the swift connection."), - cfg.StrOpt("insecure", default=False, help="Don't check SSL certificate"), - cfg.StrOpt("project_domain_id", default="default", - help="Domain ID containing project"), - cfg.StrOpt("project_domain_name", help="Domain name containing project"), - cfg.StrOpt("user_domain_id", default="default", help="User's domain id"), - cfg.StrOpt("user_domain_name", help="User's domain name"), - cfg.StrOpt("region_name", help="Region name"), - cfg.StrOpt("interface", default="publicURL", - help="The default interface for endpoint URL discovery."), -) - - -MESSAGE_SWIFT_GROUP = 'drivers:message_store:swift' - - -def _config_options(): - return [(MESSAGE_SWIFT_GROUP, MESSAGE_SWIFT_OPTIONS), ] diff --git a/zaqar/storage/utils.py b/zaqar/storage/utils.py index fbe0b5589..9130a7400 100644 --- a/zaqar/storage/utils.py +++ b/zaqar/storage/utils.py @@ -128,9 +128,9 @@ def load_storage_driver(conf, cache, storage_type=None, driver_type = 'zaqar.{0}.storage'.format(mode) - _invoke_args = [conf, cache] + _invoke_args = (conf, cache) if control_driver is not None: - _invoke_args.append(control_driver) + _invoke_args = (conf, cache, control_driver) try: mgr = driver.DriverManager(driver_type, diff --git a/zaqar/tests/base.py b/zaqar/tests/base.py index 3b27d3c7a..1e6180578 100644 --- a/zaqar/tests/base.py +++ b/zaqar/tests/base.py @@ -21,7 +21,11 @@ from oslo_log import log from osprofiler import opts import testtools -from zaqar.common import configs +from zaqar.conf import default +from zaqar.conf import drivers +from zaqar.conf import notification +from zaqar.conf import profiler +from zaqar.conf import signed_url from zaqar.tests import helpers @@ -54,18 +58,16 @@ class TestBase(testtools.TestCase): else: self.conf = cfg.ConfigOpts() - self.conf.register_opts(configs._GENERAL_OPTIONS) - self.conf.register_opts(configs._DRIVER_OPTIONS, - group=configs._DRIVER_GROUP) - self.conf.register_opts(configs._NOTIFICATION_OPTIONS, - group=configs._NOTIFICATION_GROUP) - self.conf.register_opts(configs._NOTIFICATION_OPTIONS, - group=configs._NOTIFICATION_GROUP) - self.conf.register_opts(configs._SIGNED_URL_OPTIONS, - group=configs._SIGNED_URL_GROUP) + self.conf.register_opts(default.ALL_OPTS) + self.conf.register_opts(drivers.ALL_OPTS, + group=drivers.GROUP_NAME) + self.conf.register_opts(notification.ALL_OPTS, + group=notification.GROUP_NAME) + self.conf.register_opts(signed_url.ALL_OPTS, + group=signed_url.GROUP_NAME) opts.set_defaults(self.conf) - self.conf.register_opts(configs._PROFILER_OPTIONS, - group=configs._PROFILER_GROUP) + self.conf.register_opts(profiler.ALL_OPTS, + group=profiler.GROUP_NAME) self.redis_url = os.environ.get('ZAQAR_TEST_REDIS_URL', 'redis://127.0.0.1:6379') self.mongodb_url = os.environ.get('ZAQAR_TEST_MONGODB_URL', diff --git a/zaqar/tests/functional/base.py b/zaqar/tests/functional/base.py index a9fc4305c..23f3b6068 100644 --- a/zaqar/tests/functional/base.py +++ b/zaqar/tests/functional/base.py @@ -79,8 +79,6 @@ class FunctionalTestBase(testing.TestBase): validator = validation.Validator(self.mconf) self.limits = validator._limits_conf - transport_base._config_options() - self.resource_defaults = transport_base.ResourceDefaults(self.mconf) # Always register options diff --git a/zaqar/tests/unit/common/storage/test_utils.py b/zaqar/tests/unit/common/storage/test_utils.py index ae6e4f03b..ff31ab628 100644 --- a/zaqar/tests/unit/common/storage/test_utils.py +++ b/zaqar/tests/unit/common/storage/test_utils.py @@ -15,7 +15,7 @@ import ddt -from zaqar.common import configs +from zaqar.conf import default from zaqar.storage import utils from zaqar import tests as testing @@ -25,7 +25,7 @@ class TestUtils(testing.TestBase): def setUp(self): super(TestUtils, self).setUp() - self.conf.register_opts(configs._GENERAL_OPTIONS) + self.conf.register_opts(default.ALL_OPTS) @testing.requires_mongodb def test_can_connect_succeeds_if_good_uri_mongo(self): diff --git a/zaqar/tests/unit/common/test_decorators.py b/zaqar/tests/unit/common/test_decorators.py index 90b2f0cff..d0615f532 100644 --- a/zaqar/tests/unit/common/test_decorators.py +++ b/zaqar/tests/unit/common/test_decorators.py @@ -18,8 +18,8 @@ from oslo_cache import core from oslo_config import cfg from zaqar.common import cache as oslo_cache -from zaqar.common import configs from zaqar.common import decorators +from zaqar.conf import default from zaqar.tests import base @@ -27,7 +27,7 @@ class TestDecorators(base.TestBase): def setUp(self): super(TestDecorators, self).setUp() - self.conf.register_opts(configs._GENERAL_OPTIONS) + self.conf.register_opts(default.ALL_OPTS) def test_memoized_getattr(self): diff --git a/zaqar/tests/unit/storage/test_impl_mongodb.py b/zaqar/tests/unit/storage/test_impl_mongodb.py index 6aa87984d..e5ba2c7cf 100644 --- a/zaqar/tests/unit/storage/test_impl_mongodb.py +++ b/zaqar/tests/unit/storage/test_impl_mongodb.py @@ -26,12 +26,13 @@ import six from testtools import matchers from zaqar.common import cache as oslo_cache -from zaqar.common import configs +from zaqar.conf import default +from zaqar.conf import drivers_management_store_mongodb +from zaqar.conf import drivers_message_store_mongodb from zaqar import storage from zaqar.storage import errors from zaqar.storage import mongodb from zaqar.storage.mongodb import controllers -from zaqar.storage.mongodb import options from zaqar.storage.mongodb import utils from zaqar.storage import pooling from zaqar import tests as testing @@ -51,12 +52,12 @@ class MongodbSetupMixin(object): self.driver.connection.drop_database(db) def _prepare_conf(self): - if options.MESSAGE_MONGODB_GROUP in self.conf: - self.config(options.MESSAGE_MONGODB_GROUP, + if drivers_message_store_mongodb.GROUP_NAME in self.conf: + self.config(drivers_message_store_mongodb.GROUP_NAME, database=uuid.uuid4().hex) - if options.MANAGEMENT_MONGODB_GROUP in self.conf: - self.config(options.MANAGEMENT_MONGODB_GROUP, + if drivers_management_store_mongodb.GROUP_NAME in self.conf: + self.config(drivers_management_store_mongodb.GROUP_NAME, database=uuid.uuid4().hex) @@ -67,10 +68,10 @@ class MongodbUtilsTest(MongodbSetupMixin, testing.TestBase): def setUp(self): super(MongodbUtilsTest, self).setUp() - self.conf.register_opts(options.MESSAGE_MONGODB_OPTIONS, - group=options.MESSAGE_MONGODB_GROUP) + self.conf.register_opts(drivers_message_store_mongodb.ALL_OPTS, + group=drivers_message_store_mongodb.GROUP_NAME) - self.mongodb_conf = self.conf[options.MESSAGE_MONGODB_GROUP] + self.mongodb_conf = self.conf[drivers_message_store_mongodb.GROUP_NAME] MockDriver = collections.namedtuple('MockDriver', 'mongodb_conf') @@ -152,7 +153,7 @@ class MongodbDriverTest(MongodbSetupMixin, testing.TestBase): def setUp(self): super(MongodbDriverTest, self).setUp() - self.conf.register_opts(configs._GENERAL_OPTIONS) + self.conf.register_opts(default.ALL_OPTS) self.config(unreliable=False) oslo_cache.register_config(self.conf) diff --git a/zaqar/tests/unit/storage/test_impl_redis.py b/zaqar/tests/unit/storage/test_impl_redis.py index 2ce1d073d..1837a0d6e 100644 --- a/zaqar/tests/unit/storage/test_impl_redis.py +++ b/zaqar/tests/unit/storage/test_impl_redis.py @@ -23,12 +23,12 @@ import redis from zaqar.common import cache as oslo_cache from zaqar.common import errors +from zaqar.conf import drivers_message_store_redis from zaqar import storage from zaqar.storage import pooling from zaqar.storage.redis import controllers from zaqar.storage.redis import driver from zaqar.storage.redis import messages -from zaqar.storage.redis import options from zaqar.storage.redis import utils from zaqar import tests as testing from zaqar.tests.unit.storage import base @@ -65,10 +65,10 @@ class RedisUtilsTest(testing.TestBase): def setUp(self): super(RedisUtilsTest, self).setUp() - self.conf.register_opts(options.MESSAGE_REDIS_OPTIONS, - group=options.MESSAGE_REDIS_GROUP) + self.conf.register_opts(drivers_message_store_redis.ALL_OPTS, + group=drivers_message_store_redis.GROUP_NAME) - self.redis_conf = self.conf[options.MESSAGE_REDIS_GROUP] + self.redis_conf = self.conf[drivers_message_store_redis.GROUP_NAME] MockDriver = collections.namedtuple('MockDriver', 'redis_conf') diff --git a/zaqar/tests/unit/transport/websocket/base.py b/zaqar/tests/unit/transport/websocket/base.py index 545391061..6c1f0c35f 100644 --- a/zaqar/tests/unit/transport/websocket/base.py +++ b/zaqar/tests/unit/transport/websocket/base.py @@ -15,10 +15,10 @@ from oslo_serialization import jsonutils from zaqar import bootstrap -from zaqar.common import configs +from zaqar.conf import default +from zaqar.conf import drivers_transport_websocket +from zaqar.conf import transport from zaqar import tests as testing -from zaqar.transport import validation -from zaqar.transport.websocket import driver class TestBase(testing.TestBase): @@ -31,14 +31,14 @@ class TestBase(testing.TestBase): if not self.config_file: self.skipTest("No config specified") - self.conf.register_opts(configs._GENERAL_OPTIONS) - self.conf.register_opts(validation._TRANSPORT_LIMITS_OPTIONS, - group=validation._TRANSPORT_LIMITS_GROUP) - self.transport_cfg = self.conf[validation._TRANSPORT_LIMITS_GROUP] + self.conf.register_opts(default.ALL_OPTS) + self.conf.register_opts(transport.ALL_OPTS, + group=transport.GROUP_NAME) + self.transport_cfg = self.conf[transport.GROUP_NAME] - self.conf.register_opts(driver._WS_OPTIONS, - group=driver._WS_GROUP) - self.ws_cfg = self.conf[driver._WS_GROUP] + self.conf.register_opts(drivers_transport_websocket.ALL_OPTS, + group=drivers_transport_websocket.GROUP_NAME) + self.ws_cfg = self.conf[drivers_transport_websocket.GROUP_NAME] self.conf.unreliable = True self.conf.admin_mode = True diff --git a/zaqar/tests/unit/transport/wsgi/base.py b/zaqar/tests/unit/transport/wsgi/base.py index 2b7dcc91a..df0a94124 100644 --- a/zaqar/tests/unit/transport/wsgi/base.py +++ b/zaqar/tests/unit/transport/wsgi/base.py @@ -20,10 +20,10 @@ from oslo_serialization import jsonutils from oslo_utils import uuidutils from zaqar import bootstrap -from zaqar.common import configs +from zaqar.conf import default +from zaqar.conf import drivers_transport_wsgi +from zaqar.conf import transport from zaqar import tests as testing -from zaqar.transport import validation -from zaqar.transport.wsgi import driver class TestBase(testing.TestBase): @@ -36,14 +36,14 @@ class TestBase(testing.TestBase): if not self.config_file: self.skipTest("No config specified") - self.conf.register_opts(configs._GENERAL_OPTIONS) - self.conf.register_opts(validation._TRANSPORT_LIMITS_OPTIONS, - group=validation._TRANSPORT_LIMITS_GROUP) - self.transport_cfg = self.conf[validation._TRANSPORT_LIMITS_GROUP] + self.conf.register_opts(default.ALL_OPTS) + self.conf.register_opts(transport.ALL_OPTS, + group=transport.GROUP_NAME) + self.transport_cfg = self.conf[transport.GROUP_NAME] - self.conf.register_opts(driver._WSGI_OPTIONS, - group=driver._WSGI_GROUP) - self.wsgi_cfg = self.conf[driver._WSGI_GROUP] + self.conf.register_opts(drivers_transport_wsgi.ALL_OPTS, + group=drivers_transport_wsgi.GROUP_NAME) + self.wsgi_cfg = self.conf[drivers_transport_wsgi.GROUP_NAME] self.conf.unreliable = True self.conf.admin_mode = True diff --git a/zaqar/transport/base.py b/zaqar/transport/base.py index 7662853fb..c1ef6384a 100644 --- a/zaqar/transport/base.py +++ b/zaqar/transport/base.py @@ -15,40 +15,10 @@ import abc -from oslo_config import cfg import six - -_GENERAL_TRANSPORT_OPTIONS = ( - cfg.StrOpt('auth_strategy', default='', - help=('Backend to use for authentication. ' - 'For no auth, keep it empty. ' - 'Existing strategies: keystone. ' - 'See also the keystone_authtoken section below')), -) - -_RESOURCE_DEFAULTS = ( - cfg.IntOpt('default_message_ttl', default=3600, - help=('Defines how long a message will be accessible.')), - cfg.IntOpt('default_message_delay', default=0, - help=('Defines the defautl value for queue delay seconds.' - 'The 0 means the delayed queues feature is close.')), - cfg.IntOpt('default_claim_ttl', default=300, - help=('Defines how long a message will be in claimed state.')), - cfg.IntOpt('default_claim_grace', default=60, - help=('Defines the message grace period in seconds.')), - cfg.IntOpt('default_subscription_ttl', default=3600, - help=('Defines how long a subscription will be available.')), -) - -_TRANSPORT_GROUP = 'transport' - - -def _config_options(): - return [ - (None, _GENERAL_TRANSPORT_OPTIONS), - (_TRANSPORT_GROUP, _RESOURCE_DEFAULTS), - ] +from zaqar.conf import default +from zaqar.conf import transport class ResourceDefaults(object): @@ -56,8 +26,9 @@ class ResourceDefaults(object): def __init__(self, conf): self._conf = conf - self._conf.register_opts(_RESOURCE_DEFAULTS, group=_TRANSPORT_GROUP) - self._defaults = self._conf[_TRANSPORT_GROUP] + self._conf.register_opts(transport.ALL_OPTS, + group=transport.GROUP_NAME) + self._defaults = self._conf[transport.GROUP_NAME] @property def message_ttl(self): @@ -96,7 +67,7 @@ class DriverBase(object): self._cache = cache self._control = control - self._conf.register_opts(_GENERAL_TRANSPORT_OPTIONS) + self._conf.register_opts([default.auth_strategy]) self._defaults = ResourceDefaults(self._conf) @abc.abstractmethod diff --git a/zaqar/transport/validation.py b/zaqar/transport/validation.py index f9f8fc215..396dee885 100644 --- a/zaqar/transport/validation.py +++ b/zaqar/transport/validation.py @@ -17,11 +17,11 @@ import datetime import re -from oslo_config import cfg from oslo_utils import timeutils import six from zaqar.common import consts +from zaqar.conf import transport from zaqar.i18n import _ @@ -31,70 +31,6 @@ MIN_CLAIM_GRACE = 60 MIN_DELAY_TTL = 0 MIN_SUBSCRIPTION_TTL = 60 _PURGBLE_RESOURCE_TYPES = {'messages', 'subscriptions'} - -_TRANSPORT_LIMITS_OPTIONS = ( - cfg.IntOpt('max_queues_per_page', default=20, - deprecated_name='queue_paging_uplimit', - deprecated_group='limits:transport', - help='Defines the maximum number of queues per page.'), - - cfg.IntOpt('max_messages_per_page', default=20, - deprecated_name='message_paging_uplimit', - deprecated_group='limits:transport', - help='Defines the maximum number of messages per page.'), - - cfg.IntOpt('max_subscriptions_per_page', default=20, - deprecated_name='subscription_paging_uplimit', - deprecated_group='limits:transport', - help='Defines the maximum number of subscriptions per page.'), - - cfg.IntOpt('max_messages_per_claim_or_pop', default=20, - deprecated_name='max_messages_per_claim', - help='The maximum number of messages that can be claimed (OR) ' - 'popped in a single request'), - - cfg.IntOpt('max_queue_metadata', default=64 * 1024, - deprecated_name='metadata_size_uplimit', - deprecated_group='limits:transport', - help='Defines the maximum amount of metadata in a queue.'), - - cfg.IntOpt('max_messages_post_size', default=256 * 1024, - deprecated_name='message_size_uplimit', - deprecated_group='limits:transport', - deprecated_opts=[cfg.DeprecatedOpt('max_message_size')], - help='Defines the maximum size of message posts.'), - - cfg.IntOpt('max_message_ttl', default=1209600, - deprecated_name='message_ttl_max', - deprecated_group='limits:transport', - help='Maximum amount of time a message will be available.'), - - cfg.IntOpt('max_message_delay', default=900, - help='Maximum delay seconds for messages can be claimed.'), - - cfg.IntOpt('max_claim_ttl', default=43200, - deprecated_name='claim_ttl_max', - deprecated_group='limits:transport', - help='Maximum length of a message in claimed state.'), - - cfg.IntOpt('max_claim_grace', default=43200, - deprecated_name='claim_grace_max', - deprecated_group='limits:transport', - help='Defines the maximum message grace period in seconds.'), - - cfg.ListOpt('subscriber_types', default=['http', 'https', 'mailto', - 'trust+http', 'trust+https'], - help='Defines supported subscriber types.'), - - cfg.IntOpt('max_flavors_per_page', default=20, - help='Defines the maximum number of flavors per page.'), - - cfg.IntOpt('max_pools_per_page', default=20, - help='Defines the maximum number of pools per page.'), -) - -_TRANSPORT_LIMITS_GROUP = 'transport' - # NOTE(kgriffs): Don't use \w because it isn't guaranteed to match # only ASCII characters. QUEUE_NAME_REGEX = re.compile('^[a-zA-Z0-9_\-.]+$') @@ -102,10 +38,6 @@ QUEUE_NAME_MAX_LEN = 64 PROJECT_ID_MAX_LEN = 256 -def _config_options(): - return [(_TRANSPORT_LIMITS_GROUP, _TRANSPORT_LIMITS_OPTIONS)] - - class ValidationFailed(ValueError): """User input did not follow API restrictions.""" @@ -117,9 +49,9 @@ class ValidationFailed(ValueError): class Validator(object): def __init__(self, conf): self._conf = conf - self._conf.register_opts(_TRANSPORT_LIMITS_OPTIONS, - group=_TRANSPORT_LIMITS_GROUP) - self._limits_conf = self._conf[_TRANSPORT_LIMITS_GROUP] + self._conf.register_opts(transport.ALL_OPTS, + group=transport.GROUP_NAME) + self._limits_conf = self._conf[transport.GROUP_NAME] self._supported_operations = ('add', 'remove', 'replace') def queue_identification(self, queue, project): diff --git a/zaqar/transport/websocket/driver.py b/zaqar/transport/websocket/driver.py index 77359a4e1..98595ea67 100644 --- a/zaqar/transport/websocket/driver.py +++ b/zaqar/transport/websocket/driver.py @@ -15,7 +15,6 @@ import socket -from oslo_config import cfg from oslo_log import log as logging from oslo_utils import netutils @@ -25,41 +24,16 @@ except ImportError: import trollius as asyncio from zaqar.common import decorators +from zaqar.conf import drivers_transport_websocket from zaqar.i18n import _ from zaqar.transport import base from zaqar.transport.middleware import auth from zaqar.transport.websocket import factory -_WS_OPTIONS = ( - cfg.HostAddressOpt('bind', default='127.0.0.1', - help='Address on which the self-hosting server will ' - 'listen.'), - - cfg.PortOpt('port', default=9000, - help='Port on which the self-hosting server will listen.'), - - cfg.PortOpt('external-port', - help='Port on which the service is provided to the user.'), - - cfg.HostAddressOpt('notification-bind', - help='Address on which the notification server will ' - 'listen.'), - - cfg.PortOpt('notification-port', default=0, - help='Port on which the notification server will listen.'), - -) - -_WS_GROUP = 'drivers:transport:websocket' - LOG = logging.getLogger(__name__) -def _config_options(): - return [(_WS_GROUP, _WS_OPTIONS)] - - # TODO(derekh): use escape_ipv6 from oslo.utils once available def _escape_ipv6(address): """Escape an IP address in square brackets if IPv6""" @@ -75,8 +49,9 @@ class Driver(base.DriverBase): self._api = api self._cache = cache - self._conf.register_opts(_WS_OPTIONS, group=_WS_GROUP) - self._ws_conf = self._conf[_WS_GROUP] + self._conf.register_opts(drivers_transport_websocket.ALL_OPTS, + group=drivers_transport_websocket.GROUP_NAME) + self._ws_conf = self._conf[drivers_transport_websocket.GROUP_NAME] if self._conf.auth_strategy: auth_strategy = auth.strategy(self._conf.auth_strategy) diff --git a/zaqar/transport/wsgi/driver.py b/zaqar/transport/wsgi/driver.py index 4541a8416..b93da0eb9 100644 --- a/zaqar/transport/wsgi/driver.py +++ b/zaqar/transport/wsgi/driver.py @@ -19,12 +19,12 @@ import six import socket from wsgiref import simple_server -from oslo_config import cfg from oslo_log import log as logging from oslo_utils import netutils from zaqar.common import decorators from zaqar.common.transport.wsgi import helpers +from zaqar.conf import drivers_transport_wsgi from zaqar.i18n import _ from zaqar import transport from zaqar.transport import acl @@ -37,24 +37,10 @@ from zaqar.transport.wsgi import v1_1 from zaqar.transport.wsgi import v2_0 from zaqar.transport.wsgi import version -_WSGI_OPTIONS = ( - cfg.HostAddressOpt('bind', default='127.0.0.1', - help='Address on which the self-hosting server will ' - 'listen.'), - - cfg.PortOpt('port', default=8888, - help='Port on which the self-hosting server will listen.'), -) - -_WSGI_GROUP = 'drivers:transport:wsgi' LOG = logging.getLogger(__name__) -def _config_options(): - return [(_WSGI_GROUP, _WSGI_OPTIONS)] - - class FuncMiddleware(object): def __init__(self, func): @@ -69,8 +55,9 @@ class Driver(transport.DriverBase): def __init__(self, conf, storage, cache, control): super(Driver, self).__init__(conf, storage, cache, control) - self._conf.register_opts(_WSGI_OPTIONS, group=_WSGI_GROUP) - self._wsgi_conf = self._conf[_WSGI_GROUP] + self._conf.register_opts(drivers_transport_wsgi.ALL_OPTS, + group=drivers_transport_wsgi.GROUP_NAME) + self._wsgi_conf = self._conf[drivers_transport_wsgi.GROUP_NAME] self._validate = validation.Validator(self._conf) self.app = None