From 01eff94b7625c64b3342830ea2b574be15dbef8d Mon Sep 17 00:00:00 2001 From: Erik Olof Gunnar Andersson Date: Sun, 13 Oct 2019 22:09:30 -0700 Subject: [PATCH] Centralized configuration Based upon the established conf paradigm used in Nova, this patch moves the configuration to a central location. Making config a lot easier to manage, as well a lot easier to locate. Additional changes * Fixed missing notification section in example config. * Moved external configs to generator config (e.g. osprofiler) Change-Id: I8bca356768bd710d28361b50754ddcc249a32428 --- doc/source/configuration/config.rst | 4 +- senlin/api/common/wsgi.py | 52 --- senlin/cmd/api.py | 16 +- senlin/cmd/api_wsgi.py | 8 +- senlin/cmd/conductor.py | 20 +- senlin/cmd/engine.py | 20 +- senlin/cmd/health_manager.py | 20 +- senlin/cmd/manage.py | 16 +- senlin/common/config.py | 325 ++---------------- senlin/common/profiler.py | 2 +- senlin/common/service.py | 46 ++- senlin/common/utils.py | 4 +- senlin/conf/__init__.py | 39 +++ senlin/conf/api.py | 70 ++++ senlin/conf/authentication.py | 41 +++ senlin/conf/base.py | 133 +++++++ senlin/conf/conductor.py | 36 ++ senlin/conf/dispatchers.py | 34 ++ senlin/conf/engine.py | 40 +++ senlin/conf/health_manager.py | 47 +++ senlin/conf/notification.py | 39 +++ senlin/conf/opts.py | 92 +++++ senlin/conf/receiver.py | 44 +++ senlin/conf/revision.py | 33 ++ senlin/conf/zaqar.py | 33 ++ senlin/db/sqlalchemy/api.py | 8 +- senlin/db/sqlalchemy/utils.py | 2 +- senlin/engine/environment.py | 2 +- senlin/engine/senlin_lock.py | 4 +- senlin/objects/fields.py | 2 +- senlin/objects/requests/clusters.py | 2 +- senlin/tests/unit/cmd/test_conductor.py | 16 +- senlin/tests/unit/cmd/test_engine.py | 16 +- senlin/tests/unit/cmd/test_health_manager.py | 16 +- .../unit/objects/requests/test_clusters.py | 4 +- .../tests/unit/objects/requests/test_nodes.py | 2 +- .../unit/objects/requests/test_receivers.py | 2 +- senlin/tests/unit/test_conf.py | 53 +++ setup.cfg | 4 +- tools/config-generator.conf | 3 +- 40 files changed, 891 insertions(+), 459 deletions(-) create mode 100644 senlin/conf/__init__.py create mode 100644 senlin/conf/api.py create mode 100644 senlin/conf/authentication.py create mode 100644 senlin/conf/base.py create mode 100644 senlin/conf/conductor.py create mode 100644 senlin/conf/dispatchers.py create mode 100644 senlin/conf/engine.py create mode 100644 senlin/conf/health_manager.py create mode 100644 senlin/conf/notification.py create mode 100644 senlin/conf/opts.py create mode 100644 senlin/conf/receiver.py create mode 100644 senlin/conf/revision.py create mode 100644 senlin/conf/zaqar.py create mode 100644 senlin/tests/unit/test_conf.py diff --git a/doc/source/configuration/config.rst b/doc/source/configuration/config.rst index ed43041a9..3ecfafc3e 100644 --- a/doc/source/configuration/config.rst +++ b/doc/source/configuration/config.rst @@ -20,11 +20,11 @@ Senlin uses `oslo.config` to define and manage configuration options to allow the deployer to control many aspects of the service API and the service engine. -.. show-options:: senlin.config +.. show-options:: senlin.conf Options ======= -.. currentmodule:: senlin.common.config +.. currentmodule:: senlin.conf.opts .. autofunction:: list_opts \ No newline at end of file diff --git a/senlin/api/common/wsgi.py b/senlin/api/common/wsgi.py index 0ed62868a..e2c92bc48 100644 --- a/senlin/api/common/wsgi.py +++ b/senlin/api/common/wsgi.py @@ -51,58 +51,6 @@ DEFAULT_API_VERSION = '1.0' API_VERSION_KEY = 'OpenStack-API-Version' VER_METHOD_ATTR = 'versioned_methods' -# senlin_api, api opts -api_opts = [ - cfg.IPOpt('bind_host', default='0.0.0.0', - help=_('Address to bind the server. Useful when ' - 'selecting a particular network interface.')), - cfg.PortOpt('bind_port', default=8778, - help=_('The port on which the server will listen.')), - cfg.IntOpt('backlog', default=4096, - help=_("Number of backlog requests " - "to configure the socket with.")), - cfg.StrOpt('cert_file', - help=_("Location of the SSL certificate file " - "to use for SSL mode.")), - cfg.StrOpt('key_file', - help=_("Location of the SSL key file to use " - "for enabling SSL mode.")), - cfg.IntOpt('workers', min=0, default=0, - help=_("Number of workers for Senlin service.")), - cfg.IntOpt('max_header_line', default=16384, - help=_('Maximum line size of message headers to be accepted. ' - 'max_header_line may need to be increased when using ' - 'large tokens (typically those generated by the ' - 'Keystone v3 API with big service catalogs).')), - cfg.IntOpt('tcp_keepidle', default=600, - help=_('The value for the socket option TCP_KEEPIDLE. This is ' - 'the time in seconds that the connection must be idle ' - 'before TCP starts sending keepalive probes.')), - cfg.StrOpt('api_paste_config', default="api-paste.ini", - deprecated_group='paste_deploy', - help=_("The API paste config file to use.")), - cfg.BoolOpt('wsgi_keep_alive', default=True, - deprecated_group='eventlet_opts', - help=_("If false, closes the client socket explicitly.")), - cfg.IntOpt('client_socket_timeout', default=900, - deprecated_group='eventlet_opts', - help=_("Timeout for client connections' socket operations. " - "If an incoming connection is idle for this number of " - "seconds it will be closed. A value of '0' indicates " - "waiting forever.")), - cfg.IntOpt('max_json_body_size', default=1048576, - deprecated_group='DEFAULT', - help=_('Maximum raw byte size of JSON request body.')) - -] -api_group = cfg.OptGroup('senlin_api') -cfg.CONF.register_group(api_group) -cfg.CONF.register_opts(api_opts, group=api_group) - - -def wsgi_opts(): - yield api_group.name, api_opts - def get_bind_addr(conf, default_port=None): return (conf.bind_host, conf.bind_port or default_port) diff --git a/senlin/cmd/api.py b/senlin/cmd/api.py index fbebc08c5..7c691232c 100644 --- a/senlin/cmd/api.py +++ b/senlin/cmd/api.py @@ -17,7 +17,6 @@ Senlin API Server. """ import sys -from oslo_config import cfg from oslo_log import log as logging from oslo_reports import guru_meditation_report as gmr from oslo_service import systemd @@ -27,31 +26,30 @@ from senlin.api.common import wsgi from senlin.common import config from senlin.common import messaging from senlin.common import profiler +import senlin.conf from senlin import objects from senlin import version +CONF = senlin.conf.CONF LOG = logging.getLogger('senlin.api') def main(): try: - logging.register_options(cfg.CONF) - cfg.CONF(project='senlin', prog='senlin-api', - version=version.version_info.version_string()) - config.set_config_defaults() - logging.setup(cfg.CONF, 'senlin-api') + config.parse_args(sys.argv, 'senlin-api') + logging.setup(CONF, 'senlin-api') gmr.TextGuruMeditation.setup_autorun(version) objects.register_all() messaging.setup() app = wsgi.load_paste_app() - host = cfg.CONF.senlin_api.bind_host - port = cfg.CONF.senlin_api.bind_port + host = CONF.senlin_api.bind_host + port = CONF.senlin_api.bind_port LOG.info('Starting Senlin API on %(host)s:%(port)s', {'host': host, 'port': port}) profiler.setup('senlin-api', host) - server = wsgi.Server('senlin-api', cfg.CONF.senlin_api) + server = wsgi.Server('senlin-api', CONF.senlin_api) server.start(app, default_port=port) systemd.notify_once() server.wait() diff --git a/senlin/cmd/api_wsgi.py b/senlin/cmd/api_wsgi.py index 87d3acfa1..bee95d820 100644 --- a/senlin/cmd/api_wsgi.py +++ b/senlin/cmd/api_wsgi.py @@ -16,7 +16,7 @@ Use this file for deploying senlin-api under Apache2(mode-wsgi). """ - +import sys from oslo_config import cfg from oslo_log import log as logging @@ -26,15 +26,11 @@ from senlin.common import config from senlin.common import messaging from senlin.common import profiler from senlin import objects -from senlin import version def init_app(): - logging.register_options(cfg.CONF) - cfg.CONF(project='senlin', prog='senlin-api', - version=version.version_info.version_string()) + config.parse_args(sys.argv, 'senlin-api') logging.setup(cfg.CONF, 'senlin-api') - config.set_config_defaults() objects.register_all() messaging.setup() diff --git a/senlin/cmd/conductor.py b/senlin/cmd/conductor.py index 4d86b3ccf..d8041b884 100644 --- a/senlin/cmd/conductor.py +++ b/senlin/cmd/conductor.py @@ -15,22 +15,26 @@ """ Senlin Conductor. """ -from oslo_config import cfg +import sys + from oslo_log import log as logging from oslo_reports import guru_meditation_report as gmr from oslo_service import service +from senlin.common import config from senlin.common import consts from senlin.common import messaging from senlin.common import profiler +import senlin.conf from senlin import objects from senlin import version +CONF = senlin.conf.CONF + def main(): - logging.register_options(cfg.CONF) - cfg.CONF(project='senlin', prog='senlin-conductor') - logging.setup(cfg.CONF, 'senlin-conductor') + config.parse_args(sys.argv, 'senlin-conductor') + logging.setup(CONF, 'senlin-conductor') logging.set_defaults() gmr.TextGuruMeditation.setup_autorun(version) objects.register_all() @@ -38,10 +42,10 @@ def main(): from senlin.conductor import service as conductor - profiler.setup('senlin-conductor', cfg.CONF.host) - srv = conductor.ConductorService(cfg.CONF.host, consts.CONDUCTOR_TOPIC) - launcher = service.launch(cfg.CONF, srv, - workers=cfg.CONF.conductor.workers, + profiler.setup('senlin-conductor', CONF.host) + srv = conductor.ConductorService(CONF.host, consts.CONDUCTOR_TOPIC) + launcher = service.launch(CONF, srv, + workers=CONF.conductor.workers, restart_method='mutate') # the following periodic tasks are intended serve as HA checking # srv.create_periodic_tasks() diff --git a/senlin/cmd/engine.py b/senlin/cmd/engine.py index a22d6fd5a..911e7373b 100644 --- a/senlin/cmd/engine.py +++ b/senlin/cmd/engine.py @@ -15,22 +15,26 @@ """ Senlin Engine. """ -from oslo_config import cfg +import sys + from oslo_log import log as logging from oslo_reports import guru_meditation_report as gmr from oslo_service import service +from senlin.common import config from senlin.common import consts from senlin.common import messaging from senlin.common import profiler +import senlin.conf from senlin import objects from senlin import version +CONF = senlin.conf.CONF + def main(): - logging.register_options(cfg.CONF) - cfg.CONF(project='senlin', prog='senlin-engine') - logging.setup(cfg.CONF, 'senlin-engine') + config.parse_args(sys.argv, 'senlin-engine') + logging.setup(CONF, 'senlin-engine') logging.set_defaults() gmr.TextGuruMeditation.setup_autorun(version) objects.register_all() @@ -38,10 +42,10 @@ def main(): from senlin.engine import service as engine - profiler.setup('senlin-engine', cfg.CONF.host) - srv = engine.EngineService(cfg.CONF.host, + profiler.setup('senlin-engine', CONF.host) + srv = engine.EngineService(CONF.host, consts.ENGINE_TOPIC) - launcher = service.launch(cfg.CONF, srv, - workers=cfg.CONF.engine.workers, + launcher = service.launch(CONF, srv, + workers=CONF.engine.workers, restart_method='mutate') launcher.wait() diff --git a/senlin/cmd/health_manager.py b/senlin/cmd/health_manager.py index 2abf9721c..44abab5d3 100644 --- a/senlin/cmd/health_manager.py +++ b/senlin/cmd/health_manager.py @@ -15,22 +15,26 @@ """ Senlin Health-Manager. """ -from oslo_config import cfg +import sys + from oslo_log import log as logging from oslo_reports import guru_meditation_report as gmr from oslo_service import service +from senlin.common import config from senlin.common import consts from senlin.common import messaging from senlin.common import profiler +import senlin.conf from senlin import objects from senlin import version +CONF = senlin.conf.CONF + def main(): - logging.register_options(cfg.CONF) - cfg.CONF(project='senlin', prog='senlin-health-manager') - logging.setup(cfg.CONF, 'senlin-health-manager') + config.parse_args(sys.argv, 'senlin-health-manager') + logging.setup(CONF, 'senlin-health-manager') logging.set_defaults() gmr.TextGuruMeditation.setup_autorun(version) objects.register_all() @@ -38,10 +42,10 @@ def main(): from senlin.health_manager import service as health_manager - profiler.setup('senlin-health-manager', cfg.CONF.host) - srv = health_manager.HealthManagerService(cfg.CONF.host, + profiler.setup('senlin-health-manager', CONF.host) + srv = health_manager.HealthManagerService(CONF.host, consts.HEALTH_MANAGER_TOPIC) - launcher = service.launch(cfg.CONF, srv, - workers=cfg.CONF.health_manager.workers, + launcher = service.launch(CONF, srv, + workers=CONF.health_manager.workers, restart_method='mutate') launcher.wait() diff --git a/senlin/cmd/manage.py b/senlin/cmd/manage.py index 5d98eadca..e64cd1fe9 100644 --- a/senlin/cmd/manage.py +++ b/senlin/cmd/manage.py @@ -20,11 +20,11 @@ from oslo_config import cfg from oslo_log import log as logging from oslo_utils import timeutils +from senlin.common import config from senlin.common import context from senlin.common.i18n import _ from senlin.db import api from senlin.objects import service as service_obj -from senlin import version CONF = cfg.CONF @@ -86,7 +86,7 @@ class ServiceManageCommand(object): return status = 'up' - CONF.import_opt('periodic_interval', 'senlin.common.config') + CONF.import_opt('periodic_interval', 'senlin.conf') max_interval = 2 * CONF.periodic_interval if timeutils.is_older_than(service.updated_at, max_interval): status = 'down' @@ -218,15 +218,11 @@ command_opt = cfg.SubCommandOpt('command', def main(): - logging.register_options(CONF) - logging.setup(CONF, 'senlin-manage') - CONF.register_cli_opt(command_opt) - try: - default_config_files = cfg.find_config_files('senlin', 'senlin-engine') - CONF(sys.argv[1:], project='senlin', prog='senlin-manage', - version=version.version_info.version_string(), - default_config_files=default_config_files) + CONF.register_cli_opt(command_opt) + default_config_files = cfg.find_config_files('senlin', 'senlin-manage') + config.parse_args(sys.argv, 'senlin-manage', default_config_files) + logging.setup(CONF, 'senlin-manage') except RuntimeError as e: sys.exit("ERROR: %s" % e) diff --git a/senlin/common/config.py b/senlin/common/config.py index 58e7110c7..98c78702a 100755 --- a/senlin/common/config.py +++ b/senlin/common/config.py @@ -13,318 +13,33 @@ """ Routines for configuring Senlin """ -import socket - -from keystoneauth1 import loading as ks_loading -from oslo_config import cfg +from oslo_log import log from oslo_middleware import cors -from osprofiler import opts as profiler +from oslo_utils import importutils -from senlin.api.common import wsgi -from senlin.common.i18n import _ +import senlin.conf +from senlin import version + +profiler = importutils.try_import('osprofiler.opts') + +CONF = senlin.conf.CONF -# DEFAULT, service -service_opts = [ - cfg.StrOpt('default_region_name', - help=_('Default region name used to get services endpoints.')), - cfg.IntOpt('max_response_size', - default=524288, - help=_('Maximum raw byte size of data from web response.')) -] +def parse_args(argv, name, default_config_files=None): + log.register_options(CONF) -cfg.CONF.register_opts(service_opts) + if profiler: + profiler.set_defaults(CONF) -# DEFAULT, engine -default_engine_opts = [ - cfg.IntOpt('periodic_interval', - default=60, - help=_('Seconds between running periodic tasks.')), - cfg.IntOpt('periodic_interval_max', - default=120, - help=_('Maximum seconds between periodic tasks to be called.')), - cfg.IntOpt('check_interval_max', - default=3600, - help=_('Maximum seconds between cluster check to be called.')), - cfg.IntOpt('health_check_interval_min', - default=60, - help=_('Minimum seconds between health check to be called.')), - cfg.IntOpt('periodic_fuzzy_delay', - default=10, - help=_('Range of seconds to randomly delay when starting the ' - 'periodic task scheduler to reduce stampeding. ' - '(Disable by setting to 0)')), - cfg.StrOpt('environment_dir', - default='/etc/senlin/environments', - help=_('The directory to search for environment files.')), - cfg.IntOpt('max_nodes_per_cluster', - default=1000, - help=_('Maximum nodes allowed per top-level cluster.')), - cfg.IntOpt('max_clusters_per_project', - default=100, - help=_('Maximum number of clusters any one project may have' - ' active at one time.')), - cfg.IntOpt('default_action_timeout', - default=3600, - help=_('Timeout in seconds for actions.')), - cfg.IntOpt('default_nova_timeout', - default=600, - help=_('Timeout in seconds for nova API calls.')), - cfg.IntOpt('max_actions_per_batch', - default=0, - help=_('Maximum number of node actions that each engine worker ' - 'can schedule consecutively per batch. 0 means no ' - 'limit.')), - cfg.IntOpt('batch_interval', - default=3, - help=_('Seconds to pause between scheduling two consecutive ' - 'batches of node actions.')), - cfg.IntOpt('lock_retry_times', - default=3, - help=_('Number of times trying to grab a lock.')), - cfg.IntOpt('lock_retry_interval', - default=10, - help=_('Number of seconds between lock retries.')), - cfg.IntOpt('database_retry_limit', - default=10, - help=_('Number of times retrying a failed operation on the ' - 'database.')), - cfg.IntOpt('database_retry_interval', - default=0.3, - help=_('Initial number of seconds between database retries.')), - cfg.IntOpt('database_max_retry_interval', - default=2, - help=_('Maximum number of seconds between database retries.')), - cfg.IntOpt('engine_life_check_timeout', - default=2, - help=_('RPC timeout for the engine liveness check that is used' - ' for cluster locking.')), - cfg.BoolOpt('name_unique', - default=False, - help=_('Flag to indicate whether to enforce unique names for ' - 'Senlin objects belonging to the same project.')), - cfg.IntOpt('service_down_time', - default=60, - help=_('Maximum time since last check-in for a service to be ' - 'considered up.')), - cfg.ListOpt('trust_roles', - default=[], - help=_('The roles which are delegated to the trustee by the ' - 'trustor when a cluster is created.')), -] -cfg.CONF.register_opts(default_engine_opts) + set_config_defaults() -# DEFAULT, host -rpc_opts = [ - cfg.HostAddressOpt('host', - default=socket.gethostname(), - help=_('Name of the engine node. This can be an opaque ' - 'identifier. It is not necessarily a hostname, ' - 'FQDN or IP address.')) -] -cfg.CONF.register_opts(rpc_opts) - -# DEFAULT, cloud_backend -cloud_backend_opts = [ - cfg.StrOpt('cloud_backend', default='openstack', - choices=("openstack", "openstack_test"), - help=_('Default cloud backend to use.'))] -cfg.CONF.register_opts(cloud_backend_opts) - -# DEFAULT, event dispatchers -event_opts = [ - cfg.MultiStrOpt("event_dispatchers", default=['database'], - help=_("Event dispatchers to enable."))] -cfg.CONF.register_opts(event_opts) - -# Dispatcher section -dispatcher_group = cfg.OptGroup('dispatchers') -dispatcher_opts = [ - cfg.StrOpt('priority', default='info', - choices=("critical", "error", "warning", "info", "debug"), - help=_("Lowest event priorities to be dispatched.")), - cfg.BoolOpt("exclude_derived_actions", default=True, - help=_("Exclude derived actions from events dumping.")), -] - -cfg.CONF.register_group(dispatcher_group) -cfg.CONF.register_opts(dispatcher_opts, group=dispatcher_group) - -# Authentication section -authentication_group = cfg.OptGroup('authentication') -authentication_opts = [ - cfg.StrOpt('auth_url', default='', - help=_('Complete public identity V3 API endpoint.')), - cfg.StrOpt('service_username', default='senlin', - help=_('Senlin service user name.')), - cfg.StrOpt('service_password', default='', secret=True, - help=_('Password specified for the Senlin service user.')), - cfg.StrOpt('service_project_name', default='service', - help=_('Name of the service project.')), - cfg.StrOpt('service_user_domain', default='Default', - help=_('Name of the domain for the service user.')), - cfg.StrOpt('service_project_domain', default='Default', - help=_('Name of the domain for the service project.')), -] -cfg.CONF.register_group(authentication_group) -cfg.CONF.register_opts(authentication_opts, group=authentication_group) - -# Conductor group -conductor_group = cfg.OptGroup('conductor') -conductor_opts = [ - cfg.IntOpt('workers', - default=1, - help=_('Number of senlin-conductor processes.')), - cfg.IntOpt('threads', - default=1000, - help=_('Number of senlin-conductor threads.')), -] -cfg.CONF.register_group(conductor_group) -cfg.CONF.register_opts(conductor_opts, group=conductor_group) - -# Engine group -engine_group = cfg.OptGroup('engine') -engine_opts = [ - cfg.IntOpt('workers', - default=1, - deprecated_name='num_engine_workers', - deprecated_group="DEFAULT", - help=_('Number of senlin-engine processes.')), - cfg.IntOpt('threads', - default=1000, - deprecated_name='scheduler_thread_pool_size', - deprecated_group="DEFAULT", - help=_('Number of senlin-engine threads.')), -] -cfg.CONF.register_group(engine_group) -cfg.CONF.register_opts(engine_opts, group=engine_group) - -# Health Manager Group -healthmgr_group = cfg.OptGroup('health_manager') -healthmgr_opts = [ - cfg.StrOpt('nova_control_exchange', default='nova', - help=_("Exchange name for nova notifications.")), - cfg.StrOpt('nova_notification_topic', default='versioned_notifications', - help=_("Topic name for nova notifications.")), - cfg.StrOpt('heat_control_exchange', default='heat', - help=_("Exchange name for heat notifications.")), - cfg.StrOpt('heat_notification_topic', default='notifications', - help=_("Topic name for heat notifications.")), - cfg.MultiStrOpt("enabled_endpoints", default=['nova', 'heat'], - help=_("Notification endpoints to enable.")), - cfg.IntOpt('workers', - default=1, - help=_('Number of senlin-health-manager processes.')), - cfg.IntOpt('threads', - default=1000, - deprecated_name='health_manager_thread_pool_size', - deprecated_group="DEFAULT", - help=_('Number of senlin-health-manager threads.')), -] -cfg.CONF.register_group(healthmgr_group) -cfg.CONF.register_opts(healthmgr_opts, group=healthmgr_group) - -# Revision group -revision_group = cfg.OptGroup('revision') -revision_opts = [ - cfg.StrOpt('senlin_api_revision', default='1.0', - help=_('Senlin API revision.')), - cfg.StrOpt('senlin_engine_revision', default='1.0', - help=_('Senlin engine revision.')) -] -cfg.CONF.register_group(revision_group) -cfg.CONF.register_opts(revision_opts, group=revision_group) - -# Receiver group -receiver_group = cfg.OptGroup('receiver') -receiver_opts = [ - cfg.StrOpt('host', deprecated_group='webhook', - help=_('The address for notifying and triggering receivers. ' - 'It is useful for case Senlin API service is running ' - 'behind a proxy.')), - cfg.PortOpt('port', default=8778, deprecated_group='webhook', - help=_('The port for notifying and triggering receivers. ' - 'It is useful for case Senlin API service is running ' - 'behind a proxy.')), - cfg.IntOpt('max_message_size', default=65535, - help=_('The max size(bytes) of message can be posted to ' - 'receiver queue.')) -] -cfg.CONF.register_group(receiver_group) -cfg.CONF.register_opts(receiver_opts, group=receiver_group) - -# Notification group -notification_group = cfg.OptGroup('notification') -notification_opts = [ - cfg.IntOpt('max_message_size', default=65535, - help=_('The max size(bytes) of message can be posted to ' - 'notification queue.')), - cfg.IntOpt('ttl', default=300, - help=_('The ttl in seconds of a message posted to ' - 'notification queue.')) -] -cfg.CONF.register_group(notification_group) -cfg.CONF.register_opts(notification_opts, group=notification_group) - -# Notification topic -notification_topic_opts = [ - cfg.ListOpt('notification_topics', - default=['versioned_notifications'], - help=_('Default notification topic.'))] -cfg.CONF.register_opts(notification_topic_opts) - -# Zaqar group -zaqar_group = cfg.OptGroup( - 'zaqar', title='Zaqar Options', - help=_('Configuration options for zaqar trustee.')) - -zaqar_opts = ( - ks_loading.get_auth_common_conf_options() + - ks_loading.get_auth_plugin_conf_options('password')) - -cfg.CONF.register_group(zaqar_group) -ks_loading.register_session_conf_options(cfg.CONF, 'zaqar') -ks_loading.register_auth_conf_options(cfg.CONF, 'zaqar') - -# OSProfiler -group, opts = profiler.list_opts()[0] -cfg.CONF.register_opts(opts, group=group) - - -def list_opts(): - """Return a list of oslo.config options available. - - The purpose of this function is to allow tools like the Oslo sample config - file generator to discover the options exposed to users by this service. - The returned list includes all oslo.config options which may be registered - at runtime by the service api/engine. - - Each element of the list is a tuple. The first element is the name of the - group under which the list of elements in the second element will be - registered. A group name of None corresponds to the [DEFAULT] group in - config files. - - This function is also discoverable via the 'senlin.config' entry point - under the 'oslo.config.opts' namespace. - - :returns: a list of (group_name, opts) tuples - """ - for g, o in wsgi.wsgi_opts(): - yield g, o - yield 'DEFAULT', cloud_backend_opts - yield 'DEFAULT', rpc_opts - yield 'DEFAULT', default_engine_opts - yield 'DEFAULT', service_opts - yield 'DEFAULT', event_opts - yield authentication_group.name, authentication_opts - yield conductor_group.name, conductor_opts - yield dispatcher_group.name, dispatcher_opts - yield engine_group.name, engine_opts - yield healthmgr_group.name, healthmgr_opts - yield revision_group.name, revision_opts - yield receiver_group.name, receiver_opts - yield zaqar_group.name, zaqar_opts - yield profiler.list_opts()[0] + CONF( + argv[1:], + project='senlin', + prog=name, + version=version.version_info.version_string(), + default_config_files=default_config_files, + ) def set_config_defaults(): diff --git a/senlin/common/profiler.py b/senlin/common/profiler.py index 3d0511e06..abb3e7443 100644 --- a/senlin/common/profiler.py +++ b/senlin/common/profiler.py @@ -20,7 +20,7 @@ import osprofiler.web from senlin.common import context from senlin.common import messaging -cfg.CONF.import_opt('enabled', 'senlin.common.config', group='profiler') +cfg.CONF.import_opt('enabled', 'senlin.conf', group='profiler') LOG = logging.getLogger(__name__) diff --git a/senlin/common/service.py b/senlin/common/service.py index b1257d280..e77b0f9b4 100644 --- a/senlin/common/service.py +++ b/senlin/common/service.py @@ -12,12 +12,17 @@ # License for the specific language governing permissions and limitations # under the License. - from oslo_log import log as logging from oslo_service import service +from oslo_service import sslutils +from oslo_service import wsgi +from oslo_utils import netutils + +import senlin.conf from senlin import version +CONF = senlin.conf.CONF LOG = logging.getLogger(__name__) @@ -40,3 +45,42 @@ class Service(service.Service): def stop(self, graceful=True): LOG.info('Stopping %(name)s service', {'name': self.name}) super(Service, self).stop(graceful) + + +class WSGIService(service.Service): + def __init__(self, app, name, listen, max_url_len=None): + super(WSGIService, self).__init__(CONF.senlin_api.threads) + self.app = app + self.name = name + + self.listen = listen + + self.servers = [] + + for address in self.listen: + host, port = netutils.parse_host_port(address) + server = wsgi.Server( + CONF, name, app, + host=host, + port=port, + pool_size=CONF.senlin_api.threads, + use_ssl=sslutils.is_enabled(CONF), + max_url_len=max_url_len + ) + + self.servers.append(server) + + def start(self): + for server in self.servers: + server.start() + super(WSGIService, self).start() + + def stop(self, graceful=True): + for server in self.servers: + server.stop() + super(WSGIService, self).stop(graceful) + + def wait(self): + for server in self.servers: + server.wait() + super(WSGIService, self).wait() diff --git a/senlin/common/utils.py b/senlin/common/utils.py index b7ce36198..3299a640b 100644 --- a/senlin/common/utils.py +++ b/senlin/common/utils.py @@ -32,8 +32,8 @@ from senlin.common import exception from senlin.common.i18n import _ from senlin.objects import service as service_obj -cfg.CONF.import_opt('max_response_size', 'senlin.common.config') -cfg.CONF.import_opt('periodic_interval', 'senlin.common.config') +cfg.CONF.import_opt('max_response_size', 'senlin.conf') +cfg.CONF.import_opt('periodic_interval', 'senlin.conf') LOG = logging.getLogger(__name__) _ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' diff --git a/senlin/conf/__init__.py b/senlin/conf/__init__.py new file mode 100644 index 000000000..b38a2b7c9 --- /dev/null +++ b/senlin/conf/__init__.py @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from senlin.conf import api +from senlin.conf import authentication +from senlin.conf import base +from senlin.conf import conductor +from senlin.conf import dispatchers +from senlin.conf import engine +from senlin.conf import health_manager +from senlin.conf import notification +from senlin.conf import receiver +from senlin.conf import revision +from senlin.conf import zaqar + +CONF = cfg.CONF + +api.register_opts(CONF) +authentication.register_opts(CONF) +base.register_opts(CONF) +conductor.register_opts(CONF) +dispatchers.register_opts(CONF) +engine.register_opts(CONF) +health_manager.register_opts(CONF) +notification.register_opts(CONF) +receiver.register_opts(CONF) +revision.register_opts(CONF) +zaqar.register_opts(CONF) diff --git a/senlin/conf/api.py b/senlin/conf/api.py new file mode 100644 index 000000000..ac96ed073 --- /dev/null +++ b/senlin/conf/api.py @@ -0,0 +1,70 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg + +from senlin.common.i18n import _ + + +API_GROUP = cfg.OptGroup('senlin_api') +API_OPTS = [ + cfg.IPOpt('bind_host', default='0.0.0.0', + help=_('Address to bind the server. Useful when ' + 'selecting a particular network interface.')), + cfg.PortOpt('bind_port', default=8778, + help=_('The port on which the server will listen.')), + cfg.IntOpt('backlog', default=4096, + help=_("Number of backlog requests " + "to configure the socket with.")), + cfg.StrOpt('cert_file', + help=_("Location of the SSL certificate file " + "to use for SSL mode.")), + cfg.StrOpt('key_file', + help=_("Location of the SSL key file to use " + "for enabling SSL mode.")), + cfg.IntOpt('workers', min=0, default=0, + help=_("Number of workers for Senlin service.")), + cfg.IntOpt('max_header_line', default=16384, + help=_('Maximum line size of message headers to be accepted. ' + 'max_header_line may need to be increased when using ' + 'large tokens (typically those generated by the ' + 'Keystone v3 API with big service catalogs).')), + cfg.IntOpt('tcp_keepidle', default=600, + help=_('The value for the socket option TCP_KEEPIDLE. This is ' + 'the time in seconds that the connection must be idle ' + 'before TCP starts sending keepalive probes.')), + cfg.StrOpt('api_paste_config', default="api-paste.ini", + deprecated_group='paste_deploy', + help=_("The API paste config file to use.")), + cfg.BoolOpt('wsgi_keep_alive', default=True, + deprecated_group='eventlet_opts', + help=_("If false, closes the client socket explicitly.")), + cfg.IntOpt('client_socket_timeout', default=900, + deprecated_group='eventlet_opts', + help=_("Timeout for client connections' socket operations. " + "If an incoming connection is idle for this number of " + "seconds it will be closed. A value of '0' indicates " + "waiting forever.")), + cfg.IntOpt('max_json_body_size', default=1048576, + deprecated_group='DEFAULT', + help=_('Maximum raw byte size of JSON request body.')), +] + + +def register_opts(conf): + conf.register_group(API_GROUP) + conf.register_opts(API_OPTS, group=API_GROUP) + + +def list_opts(): + return { + API_GROUP: API_OPTS, + } diff --git a/senlin/conf/authentication.py b/senlin/conf/authentication.py new file mode 100644 index 000000000..7a02e303c --- /dev/null +++ b/senlin/conf/authentication.py @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg + +from senlin.common.i18n import _ + +AUTHENTICATION_GROUP = cfg.OptGroup('authentication') +AUTHENTICATION_OPTS = [ + cfg.StrOpt('auth_url', default='', + help=_('Complete public identity V3 API endpoint.')), + cfg.StrOpt('service_username', default='senlin', + help=_('Senlin service user name.')), + cfg.StrOpt('service_password', default='', secret=True, + help=_('Password specified for the Senlin service user.')), + cfg.StrOpt('service_project_name', default='service', + help=_('Name of the service project.')), + cfg.StrOpt('service_user_domain', default='Default', + help=_('Name of the domain for the service user.')), + cfg.StrOpt('service_project_domain', default='Default', + help=_('Name of the domain for the service project.')), +] + + +def register_opts(conf): + conf.register_group(AUTHENTICATION_GROUP) + conf.register_opts(AUTHENTICATION_OPTS, group=AUTHENTICATION_GROUP) + + +def list_opts(): + return { + AUTHENTICATION_GROUP: AUTHENTICATION_OPTS + } diff --git a/senlin/conf/base.py b/senlin/conf/base.py new file mode 100644 index 000000000..a77af35e1 --- /dev/null +++ b/senlin/conf/base.py @@ -0,0 +1,133 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import socket + +from oslo_config import cfg + +from senlin.common.i18n import _ + +SENLIN_OPTS = [ + cfg.HostAddressOpt('host', + default=socket.gethostname(), + help=_('Name of the engine node. This can be an opaque ' + 'identifier. It is not necessarily a hostname, ' + 'FQDN or IP address.')), + cfg.StrOpt('default_region_name', + help=_('Default region name used to get services endpoints.')), + cfg.IntOpt('max_response_size', + default=524288, + help=_('Maximum raw byte size of data from web response.')), + cfg.ListOpt('notification_topics', + default=['versioned_notifications'], + help=_('Default notification topic.')), +] + +ENGINE_OPTS = [ + cfg.IntOpt('periodic_interval', + default=60, + help=_('Seconds between running periodic tasks.')), + cfg.IntOpt('periodic_interval_max', + default=120, + help=_('Maximum seconds between periodic tasks to be called.')), + cfg.IntOpt('check_interval_max', + default=3600, + help=_('Maximum seconds between cluster check to be called.')), + cfg.IntOpt('health_check_interval_min', + default=60, + help=_('Minimum seconds between health check to be called.')), + cfg.IntOpt('periodic_fuzzy_delay', + default=10, + help=_('Range of seconds to randomly delay when starting the ' + 'periodic task scheduler to reduce stampeding. ' + '(Disable by setting to 0)')), + cfg.StrOpt('environment_dir', + default='/etc/senlin/environments', + help=_('The directory to search for environment files.')), + cfg.IntOpt('max_nodes_per_cluster', + default=1000, + help=_('Maximum nodes allowed per top-level cluster.')), + cfg.IntOpt('max_clusters_per_project', + default=100, + help=_('Maximum number of clusters any one project may have' + ' active at one time.')), + cfg.IntOpt('default_action_timeout', + default=3600, + help=_('Timeout in seconds for actions.')), + cfg.IntOpt('default_nova_timeout', + default=600, + help=_('Timeout in seconds for nova API calls.')), + cfg.IntOpt('max_actions_per_batch', + default=0, + help=_('Maximum number of node actions that each engine worker ' + 'can schedule consecutively per batch. 0 means no ' + 'limit.')), + cfg.IntOpt('batch_interval', + default=3, + help=_('Seconds to pause between scheduling two consecutive ' + 'batches of node actions.')), + cfg.IntOpt('lock_retry_times', + default=3, + help=_('Number of times trying to grab a lock.')), + cfg.IntOpt('lock_retry_interval', + default=10, + help=_('Number of seconds between lock retries.')), + cfg.IntOpt('database_retry_limit', + default=10, + help=_('Number of times retrying a failed operation on the ' + 'database.')), + cfg.IntOpt('database_retry_interval', + default=0.3, + help=_('Initial number of seconds between database retries.')), + cfg.IntOpt('database_max_retry_interval', + default=2, + help=_('Maximum number of seconds between database retries.')), + cfg.IntOpt('engine_life_check_timeout', + default=2, + help=_('RPC timeout for the engine liveness check that is used' + ' for cluster locking.')), + cfg.BoolOpt('name_unique', + default=False, + help=_('Flag to indicate whether to enforce unique names for ' + 'Senlin objects belonging to the same project.')), + cfg.IntOpt('service_down_time', + default=60, + help=_('Maximum time since last check-in for a service to be ' + 'considered up.')), + cfg.ListOpt('trust_roles', + default=[], + help=_('The roles which are delegated to the trustee by the ' + 'trustor when a cluster is created.')), +] + +CLOUD_BACKEND_OPTS = [ + cfg.StrOpt('cloud_backend', default='openstack', + choices=("openstack", "openstack_test"), + help=_('Default cloud backend to use.')), +] + +EVENT_OPTS = [ + cfg.MultiStrOpt("event_dispatchers", default=['database'], + help=_("Event dispatchers to enable.")), +] + + +def register_opts(conf): + conf.register_opts(SENLIN_OPTS) + conf.register_opts(ENGINE_OPTS) + conf.register_opts(CLOUD_BACKEND_OPTS) + conf.register_opts(EVENT_OPTS) + + +def list_opts(): + return { + 'DEFAULT': SENLIN_OPTS + ENGINE_OPTS + CLOUD_BACKEND_OPTS + EVENT_OPTS + } diff --git a/senlin/conf/conductor.py b/senlin/conf/conductor.py new file mode 100644 index 000000000..f326f0704 --- /dev/null +++ b/senlin/conf/conductor.py @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg + +from senlin.common.i18n import _ + + +CONDUCTOR_GROUP = cfg.OptGroup('conductor') +CONDUCTOR_OPTS = [ + cfg.IntOpt('workers', + default=1, + help=_('Number of senlin-conductor processes.')), + cfg.IntOpt('threads', + default=1000, + help=_('Number of senlin-conductor threads.')), +] + + +def register_opts(conf): + conf.register_group(CONDUCTOR_GROUP) + conf.register_opts(CONDUCTOR_OPTS, group=CONDUCTOR_GROUP) + + +def list_opts(): + return { + CONDUCTOR_GROUP: CONDUCTOR_OPTS, + } diff --git a/senlin/conf/dispatchers.py b/senlin/conf/dispatchers.py new file mode 100644 index 000000000..fb8bc6023 --- /dev/null +++ b/senlin/conf/dispatchers.py @@ -0,0 +1,34 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg + +from senlin.common.i18n import _ + +DISPATCHERS_GROUP = cfg.OptGroup('dispatchers') +DISPATCHERS_OPTS = [ + cfg.StrOpt('priority', default='info', + choices=("critical", "error", "warning", "info", "debug"), + help=_("Lowest event priorities to be dispatched.")), + cfg.BoolOpt("exclude_derived_actions", default=True, + help=_("Exclude derived actions from events dumping.")), +] + + +def register_opts(conf): + conf.register_group(DISPATCHERS_GROUP) + conf.register_opts(DISPATCHERS_OPTS, group=DISPATCHERS_GROUP) + + +def list_opts(): + return { + DISPATCHERS_GROUP: DISPATCHERS_OPTS + } diff --git a/senlin/conf/engine.py b/senlin/conf/engine.py new file mode 100644 index 000000000..b743672cc --- /dev/null +++ b/senlin/conf/engine.py @@ -0,0 +1,40 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg + +from senlin.common.i18n import _ + + +ENGINE_GROUP = cfg.OptGroup('engine') +ENGINE_OPTS = [ + cfg.IntOpt('workers', + default=1, + deprecated_name='num_engine_workers', + deprecated_group="DEFAULT", + help=_('Number of senlin-engine processes.')), + cfg.IntOpt('threads', + default=1000, + deprecated_name='scheduler_thread_pool_size', + deprecated_group="DEFAULT", + help=_('Number of senlin-engine threads.')), +] + + +def register_opts(conf): + conf.register_group(ENGINE_GROUP) + conf.register_opts(ENGINE_OPTS, group=ENGINE_GROUP) + + +def list_opts(): + return { + ENGINE_GROUP: ENGINE_OPTS, + } diff --git a/senlin/conf/health_manager.py b/senlin/conf/health_manager.py new file mode 100644 index 000000000..3c671e0a2 --- /dev/null +++ b/senlin/conf/health_manager.py @@ -0,0 +1,47 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg + +from senlin.common.i18n import _ + +HEALTH_MANAGER_GROUP = cfg.OptGroup('health_manager') +HEALTH_MANAGER_OPTS = [ + cfg.StrOpt('nova_control_exchange', default='nova', + help=_("Exchange name for nova notifications.")), + cfg.StrOpt('nova_notification_topic', default='versioned_notifications', + help=_("Topic name for nova notifications.")), + cfg.StrOpt('heat_control_exchange', default='heat', + help=_("Exchange name for heat notifications.")), + cfg.StrOpt('heat_notification_topic', default='notifications', + help=_("Topic name for heat notifications.")), + cfg.MultiStrOpt("enabled_endpoints", default=['nova', 'heat'], + help=_("Notification endpoints to enable.")), + cfg.IntOpt('workers', + default=1, + help=_('Number of senlin-health-manager processes.')), + cfg.IntOpt('threads', + default=1000, + deprecated_name='health_manager_thread_pool_size', + deprecated_group="DEFAULT", + help=_('Number of senlin-health-manager threads.')), +] + + +def register_opts(conf): + conf.register_group(HEALTH_MANAGER_GROUP) + conf.register_opts(HEALTH_MANAGER_OPTS, group=HEALTH_MANAGER_GROUP) + + +def list_opts(): + return { + HEALTH_MANAGER_GROUP: HEALTH_MANAGER_OPTS + } diff --git a/senlin/conf/notification.py b/senlin/conf/notification.py new file mode 100644 index 000000000..da3866fd2 --- /dev/null +++ b/senlin/conf/notification.py @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from senlin.common.i18n import _ + +NOTIFICATION_GROUP = cfg.OptGroup( + name='notification', +) + +NOTIFICATION_OPTS = [ + cfg.IntOpt('max_message_size', default=65535, + help=_('The max size(bytes) of message can be posted to ' + 'notification queue.')), + cfg.IntOpt('ttl', default=300, + help=_('The ttl in seconds of a message posted to ' + 'notification queue.')), +] + + +def register_opts(conf): + conf.register_group(NOTIFICATION_GROUP) + conf.register_opts(NOTIFICATION_OPTS, group=NOTIFICATION_GROUP) + + +def list_opts(): + return { + NOTIFICATION_GROUP: NOTIFICATION_OPTS, + } diff --git a/senlin/conf/opts.py b/senlin/conf/opts.py new file mode 100644 index 000000000..27f1838bf --- /dev/null +++ b/senlin/conf/opts.py @@ -0,0 +1,92 @@ +# Copyright 2015 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# Copied from nova + +""" +This is the single point of entry to generate the sample configuration +file for Senlin. It collects all the necessary info from the other modules +in this package. It is assumed that: + +* every other module in this package has a 'list_opts' function which + return a dict where + * the keys are strings which are the group names + * the value of each key is a list of config options for that group +* the senlin.conf package doesn't have further packages with config options +* this module is only used in the context of sample file generation +""" + +import collections +import importlib +import os +import pkgutil + +LIST_OPTS_FUNC_NAME = "list_opts" + + +def _tupleize(dct): + """Take the dict of options and convert to the 2-tuple format.""" + return [(key, val) for key, val in dct.items()] + + +def list_opts(): + """Return a list of oslo.config options available. + + The purpose of this function is to allow tools like the Oslo sample config + file generator to discover the options exposed to users by this service. + The returned list includes all oslo.config options which may be registered + at runtime by the service api/engine. + + This function is also discoverable via the 'senlin.conf' entry point + under the 'oslo.config.opts' namespace. + + :returns: a list of (group_name, opts) tuples + """ + opts = collections.defaultdict(list) + module_names = _list_module_names() + imported_modules = _import_modules(module_names) + _append_config_options(imported_modules, opts) + return _tupleize(opts) + + +def _list_module_names(): + module_names = [] + package_path = os.path.dirname(os.path.abspath(__file__)) + for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]): + if modname == "opts" or ispkg: + continue + else: + module_names.append(modname) + return module_names + + +def _import_modules(module_names): + imported_modules = [] + for modname in module_names: + mod = importlib.import_module("senlin.conf." + modname) + if not hasattr(mod, LIST_OPTS_FUNC_NAME): + msg = ("The module 'senlin.conf.%s' should have a '%s' " + "function which returns the config options." % + (modname, LIST_OPTS_FUNC_NAME)) + raise Exception(msg) + else: + imported_modules.append(mod) + return imported_modules + + +def _append_config_options(imported_modules, config_options): + for mod in imported_modules: + configs = mod.list_opts() + for key, val in configs.items(): + config_options[key].extend(val) diff --git a/senlin/conf/receiver.py b/senlin/conf/receiver.py new file mode 100644 index 000000000..116011d5f --- /dev/null +++ b/senlin/conf/receiver.py @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from senlin.common.i18n import _ + +RECEIVER_GROUP = cfg.OptGroup( + name='receiver', +) + +RECEIVER_OPTS = [ + cfg.StrOpt('host', deprecated_group='webhook', + help=_('The address for notifying and triggering receivers. ' + 'It is useful for case Senlin API service is running ' + 'behind a proxy.')), + cfg.PortOpt('port', default=8778, deprecated_group='webhook', + help=_('The port for notifying and triggering receivers. ' + 'It is useful for case Senlin API service is running ' + 'behind a proxy.')), + cfg.IntOpt('max_message_size', default=65535, + help=_('The max size(bytes) of message can be posted to ' + 'receiver queue.')), +] + + +def register_opts(conf): + conf.register_group(RECEIVER_GROUP) + conf.register_opts(RECEIVER_OPTS, group=RECEIVER_GROUP) + + +def list_opts(): + return { + RECEIVER_GROUP: RECEIVER_OPTS, + } diff --git a/senlin/conf/revision.py b/senlin/conf/revision.py new file mode 100644 index 000000000..94e919b8b --- /dev/null +++ b/senlin/conf/revision.py @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg + +from senlin.common.i18n import _ + +REVISION_GROUP = cfg.OptGroup('revision') +REVISION_OPTS = [ + cfg.StrOpt('senlin_api_revision', default='1.0', + help=_('Senlin API revision.')), + cfg.StrOpt('senlin_engine_revision', default='1.0', + help=_('Senlin engine revision.')) +] + + +def register_opts(conf): + conf.register_group(REVISION_GROUP) + conf.register_opts(REVISION_OPTS, group=REVISION_GROUP) + + +def list_opts(): + return { + REVISION_GROUP: REVISION_OPTS + } diff --git a/senlin/conf/zaqar.py b/senlin/conf/zaqar.py new file mode 100644 index 000000000..867097390 --- /dev/null +++ b/senlin/conf/zaqar.py @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from keystoneauth1 import loading as ksa_loading +from oslo_config import cfg + +from senlin.common.i18n import _ + +ZAQAR_GROUP = cfg.OptGroup( + name='zaqar', + title=_('Configuration options for zaqar trustee.') +) + + +def register_opts(conf): + conf.register_group(ZAQAR_GROUP) + ksa_loading.register_session_conf_options(conf, ZAQAR_GROUP) + ksa_loading.register_auth_conf_options(conf, ZAQAR_GROUP) + + +def list_opts(): + return { + ZAQAR_GROUP: (ksa_loading.get_auth_common_conf_options() + + ksa_loading.get_auth_plugin_conf_options('password')) + } diff --git a/senlin/db/sqlalchemy/api.py b/senlin/db/sqlalchemy/api.py index 54e24da1b..c6c6ce91d 100755 --- a/senlin/db/sqlalchemy/api.py +++ b/senlin/db/sqlalchemy/api.py @@ -44,16 +44,16 @@ CONF = cfg.CONF _main_context_manager = None _CONTEXT = threading.local() -cfg.CONF.import_opt('database_retry_limit', 'senlin.common.config') -cfg.CONF.import_opt('database_retry_interval', 'senlin.common.config') -cfg.CONF.import_opt('database_max_retry_interval', 'senlin.common.config') +cfg.CONF.import_opt('database_retry_limit', 'senlin.conf') +cfg.CONF.import_opt('database_retry_interval', 'senlin.conf') +cfg.CONF.import_opt('database_max_retry_interval', 'senlin.conf') def _get_main_context_manager(): global _main_context_manager if not _main_context_manager: _main_context_manager = enginefacade.transaction_context() - cfg.CONF.import_group('profiler', 'senlin.common.config') + cfg.CONF.import_group('profiler', 'senlin.conf') if cfg.CONF.profiler.enabled: if cfg.CONF.profiler.trace_sqlalchemy: eng = _main_context_manager.writer.get_engine() diff --git a/senlin/db/sqlalchemy/utils.py b/senlin/db/sqlalchemy/utils.py index 0142bbbf5..6c5906ca6 100644 --- a/senlin/db/sqlalchemy/utils.py +++ b/senlin/db/sqlalchemy/utils.py @@ -78,7 +78,7 @@ def get_sort_params(value, default_key=None): def is_service_dead(service): """Check if a given service is dead.""" - cfg.CONF.import_opt("periodic_interval", "senlin.common.config") + cfg.CONF.import_opt("periodic_interval", "senlin.conf") max_elapse = 2 * cfg.CONF.periodic_interval return timeutils.is_older_than(service.updated_at, max_elapse) diff --git a/senlin/engine/environment.py b/senlin/engine/environment.py index afafde4c0..9fbca2009 100644 --- a/senlin/engine/environment.py +++ b/senlin/engine/environment.py @@ -169,7 +169,7 @@ class Environment(object): def read_global_environment(self): """Read and parse global environment files.""" - cfg.CONF.import_opt('environment_dir', 'senlin.common.config') + cfg.CONF.import_opt('environment_dir', 'senlin.conf') env_dir = cfg.CONF.environment_dir try: diff --git a/senlin/engine/senlin_lock.py b/senlin/engine/senlin_lock.py index 3657483e6..1e1c0083f 100644 --- a/senlin/engine/senlin_lock.py +++ b/senlin/engine/senlin_lock.py @@ -27,8 +27,8 @@ from senlin.objects import node_lock as nl_obj CONF = cfg.CONF -CONF.import_opt('lock_retry_times', 'senlin.common.config') -CONF.import_opt('lock_retry_interval', 'senlin.common.config') +CONF.import_opt('lock_retry_times', 'senlin.conf') +CONF.import_opt('lock_retry_interval', 'senlin.conf') LOG = logging.getLogger(__name__) diff --git a/senlin/objects/fields.py b/senlin/objects/fields.py index f704c4328..c6a176a75 100644 --- a/senlin/objects/fields.py +++ b/senlin/objects/fields.py @@ -198,7 +198,7 @@ class Capacity(fields.Integer): def __init__(self, minimum=0, maximum=None): super(Capacity, self).__init__() - CONF.import_opt("max_nodes_per_cluster", "senlin.common.config") + CONF.import_opt("max_nodes_per_cluster", "senlin.conf") if minimum > CONF.max_nodes_per_cluster: err = _("The value of 'minimum' cannot be greater than the global " diff --git a/senlin/objects/requests/clusters.py b/senlin/objects/requests/clusters.py index 54c996632..4ef30ce12 100755 --- a/senlin/objects/requests/clusters.py +++ b/senlin/objects/requests/clusters.py @@ -18,7 +18,7 @@ from senlin.objects import base from senlin.objects import fields CONF = cfg.CONF -CONF.import_opt('default_action_timeout', 'senlin.common.config') +CONF.import_opt('default_action_timeout', 'senlin.conf') @base.SenlinObjectRegistry.register diff --git a/senlin/tests/unit/cmd/test_conductor.py b/senlin/tests/unit/cmd/test_conductor.py index c07d59276..daa5ee720 100644 --- a/senlin/tests/unit/cmd/test_conductor.py +++ b/senlin/tests/unit/cmd/test_conductor.py @@ -13,6 +13,7 @@ import mock from oslo_config import cfg from senlin.cmd import conductor +from senlin.common import config from senlin.common import consts from senlin.common import messaging from senlin.common import profiler @@ -26,31 +27,26 @@ class TestConductor(base.SenlinTestCase): def setUp(self): super(TestConductor, self).setUp() - @mock.patch('oslo_config.cfg.CONF') - @mock.patch('oslo_log.log.register_options') @mock.patch('oslo_log.log.setup') @mock.patch('oslo_log.log.set_defaults') @mock.patch('oslo_service.service.launch') + @mock.patch.object(config, 'parse_args') @mock.patch.object(messaging, 'setup') @mock.patch.object(profiler, 'setup') @mock.patch.object(service, 'ConductorService') def test_main(self, mock_service, mock_profiler_setup, - mock_messaging_setup, mock_launch, - mock_log_set_defaults, mock_log_setup, - mock_register_opts, mock_conf): - mock_conf.conductor.workers = 1 - mock_conf.host = 'hostname' - + mock_messaging_setup, mock_parse_args, mock_launch, + mock_log_set_defaults, mock_log_setup): conductor.main() - mock_register_opts.assert_called_once() + mock_parse_args.assert_called_once() mock_log_setup.assert_called_once() mock_log_set_defaults.assert_called_once() mock_messaging_setup.assert_called_once() mock_profiler_setup.assert_called_once() mock_service.assert_called_once_with( - 'hostname', consts.CONDUCTOR_TOPIC + mock.ANY, consts.CONDUCTOR_TOPIC ) mock_launch.assert_called_once_with( diff --git a/senlin/tests/unit/cmd/test_engine.py b/senlin/tests/unit/cmd/test_engine.py index 054388c2f..92d164f1a 100644 --- a/senlin/tests/unit/cmd/test_engine.py +++ b/senlin/tests/unit/cmd/test_engine.py @@ -13,6 +13,7 @@ import mock from oslo_config import cfg from senlin.cmd import engine +from senlin.common import config from senlin.common import consts from senlin.common import messaging from senlin.common import profiler @@ -26,31 +27,26 @@ class TestEngine(base.SenlinTestCase): def setUp(self): super(TestEngine, self).setUp() - @mock.patch('oslo_config.cfg.CONF') - @mock.patch('oslo_log.log.register_options') @mock.patch('oslo_log.log.setup') @mock.patch('oslo_log.log.set_defaults') @mock.patch('oslo_service.service.launch') + @mock.patch.object(config, 'parse_args') @mock.patch.object(messaging, 'setup') @mock.patch.object(profiler, 'setup') @mock.patch.object(service, 'EngineService') def test_main(self, mock_service, mock_profiler_setup, - mock_messaging_setup, mock_launch, - mock_log_set_defaults, mock_log_setup, - mock_register_opts, mock_conf): - mock_conf.engine.workers = 1 - mock_conf.host = 'hostname' - + mock_messaging_setup, mock_parse_args, mock_launch, + mock_log_set_defaults, mock_log_setup): engine.main() - mock_register_opts.assert_called_once() + mock_parse_args.assert_called_once() mock_log_setup.assert_called_once() mock_log_set_defaults.assert_called_once() mock_messaging_setup.assert_called_once() mock_profiler_setup.assert_called_once() mock_service.assert_called_once_with( - 'hostname', consts.ENGINE_TOPIC + mock.ANY, consts.ENGINE_TOPIC ) mock_launch.assert_called_once_with( diff --git a/senlin/tests/unit/cmd/test_health_manager.py b/senlin/tests/unit/cmd/test_health_manager.py index c7377dd02..16b217c07 100644 --- a/senlin/tests/unit/cmd/test_health_manager.py +++ b/senlin/tests/unit/cmd/test_health_manager.py @@ -13,6 +13,7 @@ import mock from oslo_config import cfg from senlin.cmd import health_manager +from senlin.common import config from senlin.common import consts from senlin.common import messaging from senlin.common import profiler @@ -26,31 +27,26 @@ class TestHealthManager(base.SenlinTestCase): def setUp(self): super(TestHealthManager, self).setUp() - @mock.patch('oslo_config.cfg.CONF') - @mock.patch('oslo_log.log.register_options') @mock.patch('oslo_log.log.setup') @mock.patch('oslo_log.log.set_defaults') @mock.patch('oslo_service.service.launch') + @mock.patch.object(config, 'parse_args') @mock.patch.object(messaging, 'setup') @mock.patch.object(profiler, 'setup') @mock.patch.object(service, 'HealthManagerService') def test_main(self, mock_service, mock_profiler_setup, - mock_messaging_setup, mock_launch, - mock_log_set_defaults, mock_log_setup, - mock_register_opts, mock_conf): - mock_conf.health_manager.workers = 1 - mock_conf.host = 'hostname' - + mock_messaging_setup, mock_parse_args, mock_launch, + mock_log_set_defaults, mock_log_setup): health_manager.main() - mock_register_opts.assert_called_once() + mock_parse_args.assert_called_once() mock_log_setup.assert_called_once() mock_log_set_defaults.assert_called_once() mock_messaging_setup.assert_called_once() mock_profiler_setup.assert_called_once() mock_service.assert_called_once_with( - 'hostname', consts.HEALTH_MANAGER_TOPIC + mock.ANY, consts.HEALTH_MANAGER_TOPIC ) mock_launch.assert_called_once_with( diff --git a/senlin/tests/unit/objects/requests/test_clusters.py b/senlin/tests/unit/objects/requests/test_clusters.py index 750737900..749bf2e82 100755 --- a/senlin/tests/unit/objects/requests/test_clusters.py +++ b/senlin/tests/unit/objects/requests/test_clusters.py @@ -19,8 +19,8 @@ from senlin.objects.requests import clusters from senlin.tests.unit.common import base as test_base CONF = cfg.CONF -CONF.import_opt('default_action_timeout', 'senlin.common.config') -CONF.import_opt('max_nodes_per_cluster', 'senlin.common.config') +CONF.import_opt('default_action_timeout', 'senlin.conf') +CONF.import_opt('max_nodes_per_cluster', 'senlin.conf') class TestClusterCreate(test_base.SenlinTestCase): diff --git a/senlin/tests/unit/objects/requests/test_nodes.py b/senlin/tests/unit/objects/requests/test_nodes.py index ca90484a6..48f12e922 100644 --- a/senlin/tests/unit/objects/requests/test_nodes.py +++ b/senlin/tests/unit/objects/requests/test_nodes.py @@ -17,7 +17,7 @@ from senlin.objects.requests import nodes from senlin.tests.unit.common import base as test_base CONF = cfg.CONF -CONF.import_opt('default_action_timeout', 'senlin.common.config') +CONF.import_opt('default_action_timeout', 'senlin.conf') class TestNodeCreate(test_base.SenlinTestCase): diff --git a/senlin/tests/unit/objects/requests/test_receivers.py b/senlin/tests/unit/objects/requests/test_receivers.py index 6c77e1b12..c6b21eb98 100644 --- a/senlin/tests/unit/objects/requests/test_receivers.py +++ b/senlin/tests/unit/objects/requests/test_receivers.py @@ -20,7 +20,7 @@ from senlin.objects.requests import receivers from senlin.tests.unit.common import base as test_base CONF = cfg.CONF -CONF.import_opt('default_action_timeout', 'senlin.common.config') +CONF.import_opt('default_action_timeout', 'senlin.conf') class TestReceiverCreate(test_base.SenlinTestCase): diff --git a/senlin/tests/unit/test_conf.py b/senlin/tests/unit/test_conf.py new file mode 100644 index 000000000..fa345e294 --- /dev/null +++ b/senlin/tests/unit/test_conf.py @@ -0,0 +1,53 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import mock +import oslotest.base + +from senlin.conf import engine +from senlin.conf import opts + + +class TestConfOpts(oslotest.base.BaseTestCase): + def setUp(self): + super(TestConfOpts, self).setUp() + + def test_opts_tupleize(self): + self.assertEqual([('a', 'b')], opts._tupleize({'a': 'b'})) + + def test_opts_list(self): + self.assertIsInstance(opts.list_opts(), list) + + @mock.patch('pkgutil.iter_modules') + def test_opts_list_module_names(self, mock_iter_modules): + mock_iter_modules.return_value = iter( + [ + (None, 'api', False), + (None, 'authentication', False), + (None, 'unknown', True), + ] + ) + + self.assertEqual(['api', 'authentication'], opts._list_module_names()) + + def test_opts_import_modules(self): + self.assertEqual([engine], opts._import_modules(['engine'])) + + @mock.patch('importlib.import_module') + def test_opts_import_invalid_module(self, mock_import_module): + mock_import_module.return_value = None + + self.assertRaisesRegex( + Exception, + "The module 'senlin.conf.invalid' should have a 'list_opts' " + "function which returns the config options.", + opts._import_modules, ['invalid'] + ) diff --git a/setup.cfg b/setup.cfg index ff4654ddc..a67f46c66 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,10 +37,10 @@ wsgi_scripts = senlin-wsgi-api = senlin.cmd.api_wsgi:init_app oslo.config.opts = - senlin.config = senlin.common.config:list_opts + senlin.conf = senlin.conf.opts:list_opts oslo.config.opts.defaults = - senlin.config = senlin.common.config:set_config_defaults + senlin.conf = senlin.common.config:set_config_defaults oslo.policy.policies = senlin = senlin.common.policies:list_rules diff --git a/tools/config-generator.conf b/tools/config-generator.conf index d0c833f54..9163f86df 100644 --- a/tools/config-generator.conf +++ b/tools/config-generator.conf @@ -1,7 +1,7 @@ [DEFAULT] output_file = etc/senlin/senlin.conf.sample wrap_width = 119 -namespace = senlin.config +namespace = senlin.conf namespace = keystonemiddleware.auth_token namespace = oslo.db namespace = oslo.log @@ -12,3 +12,4 @@ namespace = oslo.policy namespace = oslo.service.periodic_task namespace = oslo.service.service namespace = oslo.service.sslutils +namespace = osprofiler