From bc5fa37b20a937b74554f24af7e25e6203d09392 Mon Sep 17 00:00:00 2001 From: Kiall Mac Innes Date: Tue, 8 Jan 2013 11:06:56 +0000 Subject: [PATCH] Switch to a single config file, using a config group per service. This has two advantages: - Avoids Naming Conflicts (e.g. backend-driver) - Allows common options to be specified once for all services (e.g. pybasedir, logdir) Fixs bug #1096850 Change-Id: I5b02f591ccea6d1a8201b21c3cb8f92bcf6b30fa --- bin/moniker-agent | 2 +- bin/moniker-api | 2 +- bin/moniker-central | 2 +- bin/moniker-manage | 6 ++- etc/moniker-agent.conf.sample | 39 ---------------- etc/moniker-api.conf.sample | 26 ----------- ...entral.conf.sample => moniker.conf.sample} | 44 +++++++++++++++++-- moniker/agent/__init__.py | 25 +++++++++++ moniker/agent/service.py | 7 +-- moniker/api/__init__.py | 5 ++- moniker/api/auth.py | 2 +- moniker/api/service.py | 7 +-- moniker/central/__init__.py | 29 ++++++++++++ moniker/central/service.py | 18 +++----- moniker/manage/database.py | 5 --- moniker/storage/__init__.py | 2 +- moniker/tests/__init__.py | 31 +++++++++++-- moniker/tests/test_agent/__init__.py | 20 +++++++++ moniker/tests/test_agent/test_service.py | 29 ++++++++++++ moniker/tests/test_backend/__init__.py | 2 +- moniker/tests/test_backend/test_bind9.py | 2 +- moniker/tests/test_backend/test_fake.py | 2 +- moniker/tests/test_backend/test_mysqlbind9.py | 2 +- moniker/tests/test_central/test_service.py | 5 +++ 24 files changed, 205 insertions(+), 109 deletions(-) delete mode 100644 etc/moniker-agent.conf.sample delete mode 100644 etc/moniker-api.conf.sample rename etc/{moniker-central.conf.sample => moniker.conf.sample} (63%) create mode 100644 moniker/tests/test_agent/__init__.py create mode 100644 moniker/tests/test_agent/test_service.py diff --git a/bin/moniker-agent b/bin/moniker-agent index e3e327cf4..4845b2a1c 100755 --- a/bin/moniker-agent +++ b/bin/moniker-agent @@ -23,7 +23,7 @@ from moniker.agent import service as agent_service eventlet.monkey_patch() -utils.read_config('moniker-agent', sys.argv) +utils.read_config('moniker', sys.argv) logging.setup('moniker') diff --git a/bin/moniker-api b/bin/moniker-api index a7534dc39..15eb2bff0 100755 --- a/bin/moniker-api +++ b/bin/moniker-api @@ -23,7 +23,7 @@ from moniker.api import service as api_service eventlet.monkey_patch() -utils.read_config('moniker-api', sys.argv) +utils.read_config('moniker', sys.argv) logging.setup('moniker') diff --git a/bin/moniker-central b/bin/moniker-central index 8039598f5..a6c3c1048 100755 --- a/bin/moniker-central +++ b/bin/moniker-central @@ -23,7 +23,7 @@ from moniker.central import service as central_service eventlet.monkey_patch() -utils.read_config('moniker-central', sys.argv) +utils.read_config('moniker', sys.argv) logging.setup('moniker') diff --git a/bin/moniker-manage b/bin/moniker-manage index 6e91b4faa..abaea132a 100755 --- a/bin/moniker-manage +++ b/bin/moniker-manage @@ -15,7 +15,11 @@ # License for the specific language governing permissions and limitations # under the License. import sys +from moniker import utils from moniker.manage import MonikerShell +# TODO: Sypport passing --config-file and --config-dir to read_config +utils.read_config('moniker', []) + shell = MonikerShell() -sys.exit(shell.run(sys.argv)) +sys.exit(shell.run(sys.argv[1:])) diff --git a/etc/moniker-agent.conf.sample b/etc/moniker-agent.conf.sample deleted file mode 100644 index 0725a4976..000000000 --- a/etc/moniker-agent.conf.sample +++ /dev/null @@ -1,39 +0,0 @@ -[DEFAULT] -# Show more verbose log output (sets INFO log level output) -verbose = True - -# Show debugging output in logs (sets DEBUG log level output) -debug = False - -# Top-level directory for maintaining moniker's state -#state_path = /var/lib/moniker - -# Log directory -#logdir=/var/log/moniker - -# Driver used for backend communication (e.g. bind9, powerdns) -#backend_driver=bind9 - -# There has to be a better way to set these defaults -allowed_rpc_exception_modules = moniker.exceptions, moniker.openstack.common.exception -logging_context_format_string = %(asctime)s %(levelname)s %(name)s [%(request_id)s %(user)s %(tenant)s] %(instance)s %(message)s -default_log_levels = amqplib=WARN, sqlalchemy=WARN, boto=WARN, suds=INFO, keystone=INFO, eventlet.wsgi.server=WARN, stevedore=WARN - -# Ability to configure each backend individually -#[backend:bind9] -#rndc_path = /usr/sbin/rndc -#rndc_host = 127.0.0.1 -#rndc_port = 953 -#rndc_config_file = /etc/rndc.conf -#rndc_key_file = /etc/rndc.key - -# MySQLBind agent options -#[backend:mysqlbind9] -#database_connection = mysql://user:password@host/schema -#rndc_path = /usr/sbin/rndc -#rndc_host = 127.0.0.1 -#rndc_port = 953 -#rndc_config_file = /etc/rndc.conf -#rndc_key_file = /etc/rndc.key -#write_database = True -#dns_server_type = master diff --git a/etc/moniker-api.conf.sample b/etc/moniker-api.conf.sample deleted file mode 100644 index 44f45cdc9..000000000 --- a/etc/moniker-api.conf.sample +++ /dev/null @@ -1,26 +0,0 @@ -[DEFAULT] -# Show more verbose log output (sets INFO log level output) -verbose = True - -# Show debugging output in logs (sets DEBUG log level output) -debug = False - -# Top-level directory for maintaining moniker's state -#state_path = /var/lib/moniker - -# Log directory -#logdir=/var/log/moniker - -# Address to bind the API server -api_host = 0.0.0.0 - -# Port the bind the API server to -api_port = 9001 - -# Authentication strategy to use - can be either "noauth" or "keystone" -# auth_strategy = noauth - -# There has to be a better way to set these defaults -allowed_rpc_exception_modules = moniker.exceptions, moniker.openstack.common.exception -logging_context_format_string = %(asctime)s %(levelname)s %(name)s [%(request_id)s %(user)s %(tenant)s] %(instance)s %(message)s -default_log_levels = amqplib=WARN, sqlalchemy=WARN, boto=WARN, suds=INFO, keystone=INFO, eventlet.wsgi.server=WARN, stevedore=WARN diff --git a/etc/moniker-central.conf.sample b/etc/moniker.conf.sample similarity index 63% rename from etc/moniker-central.conf.sample rename to etc/moniker.conf.sample index 05a78bddc..94fb6e80c 100644 --- a/etc/moniker-central.conf.sample +++ b/etc/moniker.conf.sample @@ -1,4 +1,5 @@ [DEFAULT] +## General Configuration # Show more verbose log output (sets INFO log level output) verbose = True @@ -11,9 +12,6 @@ debug = False # Log directory #logdir=/var/log/moniker -# Driver used for backend communication (e.g. rpc, bind9, powerdns) -#backend_driver=rpc - # There has to be a better way to set these defaults allowed_rpc_exception_modules = moniker.exceptions, moniker.openstack.common.exception logging_context_format_string = %(asctime)s %(levelname)s %(name)s [%(request_id)s %(user)s %(tenant)s] %(instance)s %(message)s @@ -22,11 +20,30 @@ default_log_levels = amqplib=WARN, sqlalchemy=WARN, boto=WARN, suds=INFO, keysto # Driver used for issuing notifications #notification_driver=moniker.openstack.common.notifier.rabbit_notifier +## Service Configuration +[service:central] + +# Driver used for backend communication (e.g. rpc, bind9, powerdns) +#backend_driver=rpc + # List of notification handlers to enable, configuration of these needs to # correspond to a [handler:my_driver] section below or else in the config #enabled_notification_handlers = nova_fixed -# Sections for *SQL storages +[service:api] +# Address to bind the API server +api_host = 0.0.0.0 + +# Port the bind the API server to +api_port = 9001 +# Authentication strategy to use - can be either "noauth" or "keystone" +# auth_strategy = noauth + +[service:agent] +# Driver used for backend communication (e.g. bind9, powerdns) +#backend_driver=bind9 + +## Storage Configuration #[storage:sqlalchemy] # Database connection string - to configure options for a given implementation # like sqlalchemy or other see below @@ -38,7 +55,26 @@ default_log_levels = amqplib=WARN, sqlalchemy=WARN, boto=WARN, suds=INFO, keysto #max_retries = 10 #retry_interval = 10 +## Notification Handler Configuration #[handler:nova_fixed] #domain_id = #notification_topics = monitor #control_exchange = 'nova' + +## Backend Configuration +#[backend:bind9] +#rndc_path = /usr/sbin/rndc +#rndc_host = 127.0.0.1 +#rndc_port = 953 +#rndc_config_file = /etc/rndc.conf +#rndc_key_file = /etc/rndc.key + +#[backend:mysqlbind9] +#database_connection = mysql://user:password@host/schema +#rndc_path = /usr/sbin/rndc +#rndc_host = 127.0.0.1 +#rndc_port = 953 +#rndc_config_file = /etc/rndc.conf +#rndc_key_file = /etc/rndc.key +#write_database = True +#dns_server_type = master diff --git a/moniker/agent/__init__.py b/moniker/agent/__init__.py index e69de29bb..aa4a3d75b 100644 --- a/moniker/agent/__init__.py +++ b/moniker/agent/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2012 Hewlett-Packard Development Company, L.P. All Rights Reserved. +# +# Author: Kiall Mac Innes +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from moniker.openstack.common import cfg + +cfg.CONF.register_group(cfg.OptGroup( + name='service:agent', title="Configuration for Agent Service" +)) + +cfg.CONF.register_opts([ + cfg.StrOpt('backend-driver', default='bind9', + help='The backend driver to use'), +], group='service:agent') diff --git a/moniker/agent/service.py b/moniker/agent/service.py index 7cf969482..33e1ed14e 100644 --- a/moniker/agent/service.py +++ b/moniker/agent/service.py @@ -20,15 +20,10 @@ from moniker import backend LOG = logging.getLogger(__name__) -cfg.CONF.register_opts([ - cfg.StrOpt('backend-driver', default='bind9', - help='The backend driver to use'), -]) - class Service(rpc_service.Service): def __init__(self, *args, **kwargs): - manager = backend.get_backend(cfg.CONF.backend_driver) + manager = backend.get_backend(cfg.CONF['service:agent'].backend_driver) kwargs.update( host=cfg.CONF.host, diff --git a/moniker/api/__init__.py b/moniker/api/__init__.py index 71c90fe65..cb4d97725 100644 --- a/moniker/api/__init__.py +++ b/moniker/api/__init__.py @@ -17,6 +17,9 @@ import flask from moniker.openstack.common import cfg from moniker.openstack.common import jsonutils as json +cfg.CONF.register_group(cfg.OptGroup( + name='service:api', title="Configuration for API Service" +)) cfg.CONF.register_opts([ cfg.StrOpt('api_host', default='0.0.0.0', @@ -28,7 +31,7 @@ cfg.CONF.register_opts([ cfg.StrOpt('auth_strategy', default='noauth', help='The strategy to use for auth. Supports noauth or ' 'keystone'), -]) +], group='service:api') # Allows us to serialize datetime's etc diff --git a/moniker/api/auth.py b/moniker/api/auth.py index 80094526d..4bef3cc91 100644 --- a/moniker/api/auth.py +++ b/moniker/api/auth.py @@ -27,7 +27,7 @@ def pipeline_factory(loader, global_conf, **local_conf): Code nabbed from cinder. """ - pipeline = local_conf[cfg.CONF.auth_strategy] + pipeline = local_conf[cfg.CONF['service:api'].auth_strategy] pipeline = pipeline.split() filters = [loader.get_filter(n) for n in pipeline[:-1]] app = loader.get_app(pipeline[-1]) diff --git a/moniker/api/service.py b/moniker/api/service.py index 793b64a54..6bfa858dc 100644 --- a/moniker/api/service.py +++ b/moniker/api/service.py @@ -27,7 +27,8 @@ LOG = logging.getLogger(__name__) class Service(wsgi.Service): def __init__(self, backlog=128, threads=1000): - config_paths = utils.find_config(cfg.CONF.api_paste_config) + api_paste_config = cfg.CONF['service:api'].api_paste_config + config_paths = utils.find_config(api_paste_config) if len(config_paths) == 0: msg = 'Unable to determine appropriate api-paste-config file' @@ -39,7 +40,7 @@ class Service(wsgi.Service): name='osapi_dns') super(Service, self).__init__(application=application, - host=cfg.CONF.api_host, - port=cfg.CONF.api_port, + host=cfg.CONF['service:api'].api_host, + port=cfg.CONF['service:api'].api_port, backlog=backlog, threads=threads) diff --git a/moniker/central/__init__.py b/moniker/central/__init__.py index e69de29bb..a7d875158 100644 --- a/moniker/central/__init__.py +++ b/moniker/central/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2012 Hewlett-Packard Development Company, L.P. All Rights Reserved. +# +# Author: Kiall Mac Innes +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from moniker.openstack.common import cfg + +cfg.CONF.register_group(cfg.OptGroup( + name='service:central', title="Configuration for Central Service" +)) + +cfg.CONF.register_opts([ + cfg.StrOpt('backend-driver', default='rpc', + help='The backend driver to use'), + cfg.StrOpt('storage-driver', default='sqlalchemy', + help='The storage driver to use'), + cfg.ListOpt('enabled-notification-handlers', default=[], + help='Enabled Notification Handlers'), +], group='service:central') diff --git a/moniker/central/service.py b/moniker/central/service.py index 32ba96169..954c73f39 100644 --- a/moniker/central/service.py +++ b/moniker/central/service.py @@ -27,20 +27,12 @@ LOG = logging.getLogger(__name__) HANDLER_NAMESPACE = 'moniker.notification.handler' -cfg.CONF.register_opts([ - cfg.StrOpt('backend-driver', default='rpc', - help='The backend driver to use'), - cfg.StrOpt('storage-driver', default='sqlalchemy', - help='The storage driver to use'), - cfg.ListOpt('enabled-notification-handlers', default=[], - help='Enabled Notification Handlers'), -]) - class Service(rpc_service.Service): def __init__(self, *args, **kwargs): - self.backend = backend.get_backend(cfg.CONF.backend_driver) + backend_driver = cfg.CONF['service:central'].backend_driver + self.backend = backend.get_backend(backend_driver) kwargs.update( host=cfg.CONF.host, @@ -63,12 +55,14 @@ class Service(rpc_service.Service): def _init_extensions(self): """ Loads and prepares all enabled extensions """ + enabled_notification_handlers = \ + cfg.CONF['service:central'].enabled_notification_handlers + self.extensions_manager = NamedExtensionManager( - HANDLER_NAMESPACE, names=cfg.CONF.enabled_notification_handlers) + HANDLER_NAMESPACE, names=enabled_notification_handlers) def _load_extension(ext): handler_cls = ext.plugin - handler_cls.register_opts(cfg.CONF) return handler_cls(central_service=self) try: diff --git a/moniker/manage/database.py b/moniker/manage/database.py index fe7c6543d..2c25b80b9 100644 --- a/moniker/manage/database.py +++ b/moniker/manage/database.py @@ -19,7 +19,6 @@ from migrate.versioning import api as versioning_api from cliff.command import Command from moniker.openstack.common import log as logging from moniker.openstack.common import cfg -from moniker import utils LOG = logging.getLogger(__name__) REPOSITORY = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', @@ -33,8 +32,6 @@ class InitCommand(Command): "Init database" def take_action(self, parsed_args): - utils.read_config('moniker-central', []) - url = cfg.CONF['storage:sqlalchemy'].database_connection if not os.path.exists(REPOSITORY): @@ -53,8 +50,6 @@ class SyncCommand(Command): def take_action(self, parsed_args): # TODO: Support specifying version - utils.read_config('moniker-central', []) - url = cfg.CONF['storage:sqlalchemy'].database_connection if not os.path.exists(REPOSITORY): diff --git a/moniker/storage/__init__.py b/moniker/storage/__init__.py index 88051886c..d08c87277 100644 --- a/moniker/storage/__init__.py +++ b/moniker/storage/__init__.py @@ -28,7 +28,7 @@ def get_engine(engine_name): def get_connection(): - engine = get_engine(cfg.CONF.storage_driver) + engine = get_engine(cfg.CONF['service:central'].storage_driver) return engine.get_connection() diff --git a/moniker/tests/__init__.py b/moniker/tests/__init__.py index 574c09006..139f0247b 100644 --- a/moniker/tests/__init__.py +++ b/moniker/tests/__init__.py @@ -20,11 +20,18 @@ from moniker.openstack.common import cfg from moniker.openstack.common import log as logging from moniker.context import MonikerContext from moniker import storage +from moniker.agent import service as agent_service from moniker.api import service as api_service from moniker.central import service as central_service LOG = logging.getLogger(__name__) +cfg.CONF.import_opt('storage_driver', 'moniker.central', + group='service:central') +cfg.CONF.import_opt('backend_driver', 'moniker.agent', + group='service:agent') +cfg.CONF.import_opt('auth_strategy', 'moniker.api', + group='service:api') cfg.CONF.import_opt('database_connection', 'moniker.storage.impl_sqlalchemy', group='storage:sqlalchemy') @@ -98,18 +105,33 @@ class TestCase(unittest2.TestCase, AssertMixin): super(TestCase, self).setUp() self.mox = mox.Mox() + + self.config( + notification_driver=[], + rpc_backend='moniker.openstack.common.rpc.impl_fake', + ) + self.config( storage_driver='sqlalchemy', backend_driver='fake', - notification_driver=[], - rpc_backend='moniker.openstack.common.rpc.impl_fake', - auth_strategy='noauth' + group='service:central' + ) + + self.config( + backend_driver='fake', + group='service:agent' + ) + + self.config( + auth_strategy='noauth', + group='service:api' ) self.config( database_connection='sqlite://', group='storage:sqlalchemy' ) + storage.setup_schema() self.admin_context = self.get_admin_context() @@ -126,6 +148,9 @@ class TestCase(unittest2.TestCase, AssertMixin): for k, v in kwargs.iteritems(): cfg.CONF.set_override(k, v, group) + def get_agent_service(self): + return agent_service.Service() + def get_api_service(self): return api_service.Service() diff --git a/moniker/tests/test_agent/__init__.py b/moniker/tests/test_agent/__init__.py new file mode 100644 index 000000000..455509cd5 --- /dev/null +++ b/moniker/tests/test_agent/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2012 Hewlett-Packard Development Company, L.P. All Rights Reserved. +# +# Author: Kiall Mac Innes +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from moniker.tests import TestCase + + +class AgentTestCase(TestCase): + __test__ = False diff --git a/moniker/tests/test_agent/test_service.py b/moniker/tests/test_agent/test_service.py new file mode 100644 index 000000000..453f17ec9 --- /dev/null +++ b/moniker/tests/test_agent/test_service.py @@ -0,0 +1,29 @@ +# Copyright 2012 Hewlett-Packard Development Company, L.P. All Rights Reserved. +# +# Author: Kiall Mac Innes +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from moniker.tests.test_agent import AgentTestCase + + +class AgentServiceTest(AgentTestCase): + __test__ = True + + def setUp(self): + super(AgentServiceTest, self).setUp() + self.service = self.get_agent_service() + + def test_start_and_stop(self): + # Ensures the start/stop actions don't raise + self.service.start() + self.service.stop() diff --git a/moniker/tests/test_backend/__init__.py b/moniker/tests/test_backend/__init__.py index 137ad8608..4fb1d6a29 100644 --- a/moniker/tests/test_backend/__init__.py +++ b/moniker/tests/test_backend/__init__.py @@ -25,7 +25,7 @@ class BackendTestCase(TestCase): __test__ = False def get_backend_driver(self): - return backend.get_backend(cfg.CONF.backend_driver) + return backend.get_backend(cfg.CONF['service:agent'].backend_driver) def test_constructor(self): self.get_backend_driver() diff --git a/moniker/tests/test_backend/test_bind9.py b/moniker/tests/test_backend/test_bind9.py index 97a59fd49..c27319586 100644 --- a/moniker/tests/test_backend/test_bind9.py +++ b/moniker/tests/test_backend/test_bind9.py @@ -25,4 +25,4 @@ class Bind9BackendDriverTestCase(BackendTestCase): def setUp(self): super(Bind9BackendDriverTestCase, self).setUp() - self.config(backend_driver='bind9') + self.config(backend_driver='bind9', group='service:agent') diff --git a/moniker/tests/test_backend/test_fake.py b/moniker/tests/test_backend/test_fake.py index e1d7539a3..ff4d89e20 100644 --- a/moniker/tests/test_backend/test_fake.py +++ b/moniker/tests/test_backend/test_fake.py @@ -25,4 +25,4 @@ class FakeBackendDriverTestCase(BackendTestCase): def setUp(self): super(FakeBackendDriverTestCase, self).setUp() - self.config(backend_driver='fake') + self.config(backend_driver='fake', group='service:agent') diff --git a/moniker/tests/test_backend/test_mysqlbind9.py b/moniker/tests/test_backend/test_mysqlbind9.py index deae6d7f9..f6ea44435 100644 --- a/moniker/tests/test_backend/test_mysqlbind9.py +++ b/moniker/tests/test_backend/test_mysqlbind9.py @@ -25,4 +25,4 @@ class MySQLBind9BackendDriverTestCase(BackendTestCase): def setUp(self): super(MySQLBind9BackendDriverTestCase, self).setUp() - self.config(backend_driver='mysqlbind9') + self.config(backend_driver='mysqlbind9', group='service:agent') diff --git a/moniker/tests/test_central/test_service.py b/moniker/tests/test_central/test_service.py index 4ed6f6890..380703dda 100644 --- a/moniker/tests/test_central/test_service.py +++ b/moniker/tests/test_central/test_service.py @@ -28,6 +28,11 @@ class CentralServiceTest(CentralTestCase): super(CentralServiceTest, self).setUp() self.central_service = self.get_central_service() + def test_start_and_stop(self): + # Ensures the start/stop actions don't raise + self.central_service.start() + self.central_service.stop() + # Server Tests def test_create_server(self): context = self.get_admin_context()