feat: separate config for queues and proxy

This patchset separates the configuration of the proxy from that of
the queues server. This was done in order to simplify the
configuration file for each, and because it is not expected that the
proxy and the queues servers would be launched on the same
host. Furthermore, many of the proxy options are not relevant to the
queues server.

Furthermore, to allow this, common.config had to be modified to take a
prog parameter. This enabled the ability to save multiple
configuration files to one directory. See below for details.

The new files are:
- etc/marconi-proxy.conf
- etc/marconi-queues.conf

They are expected to be saved to one of:
- ~/.marconi
- /etc/marconi

Regarding namespaces, queues specific options are associated with the
'queues:*' group and proxy specific options are associated to the
'proxy:*' group.

The appropriate changes are also applied to the test suite and
helpers.

Change-Id: I7cf25e47ecff47934b50c21000b31308e1a4c8a9
Implements: blueprint placement-service

etc/marconi-proxy.conf-sample

@@ -0,0 +1,43 @@
+# By default, this should line in one of:
+# ~/.marconi/marconi-proxy.conf
+# /etc/marconi/marconi-proxy.conf
+
+[DEFAULT]
+# Show more verbose log output (sets INFO log level output)
+;verbose = False
+
+# Show debugging output in logs (sets DEBUG log level output)
+;debug = False
+
+# Log to this file!
+log_file = /var/log/marconi/proxy.log
+
+;auth_strategy =
+
+# ================= Syslog Options ============================
+
+# Send logs to syslog (/dev/log) instead of to file specified
+# by `log_file`
+;use_syslog = False
+
+# Facility to use. If unset defaults to LOG_USER.
+;syslog_log_facility = LOG_LOCAL0
+
+
+# Transport driver module (e.g., wsgi, zmq)
+# Storage driver module (e.g., mongodb, sqlite)
+[proxy:drivers]
+transport = wsgi
+storage = mongodb
+
+[proxy:drivers:transport:wsgi]
+;bind = 0.0.0.0
+;port = 8889
+
+[proxy:drivers:storage:mongodb]
+uri = mongodb://db1.example.net,db2.example.net:2500/?replicaSet=test&ssl=true&w=majority
+database = marconi_proxy
+
+[oslo_cache]
+;cache_backend = memcached
+;cache_prefix = my_namespace

etc/marconi-queues.conf-sample

@@ -1,3 +1,7 @@
+# By default, this should line in one of:
+# ~/.marconi/marconi-queues.conf
+# /etc/marconi/marconi-queues.conf
+
 [DEFAULT]
 # Show more verbose log output (sets INFO log level output)
 ;verbose = False
@@ -6,7 +10,7 @@
 ;debug = False
 
 # Log to this file!
-log_file = /var/log/marconi/server.log
+log_file = /var/log/marconi/queues.log
 
 ;auth_strategy =
 
@@ -20,19 +24,14 @@ log_file = /var/log/marconi/server.log
 ;syslog_log_facility = LOG_LOCAL0
 
 
-[drivers]
+[queues:drivers]
 # Transport driver module (e.g., wsgi, zmq)
 transport = wsgi
 
 # Storage driver module (e.g., mongodb, sqlite)
 storage = mongodb
 
-# transport and storage drivers for use with marconi-proxy
+[queues:drivers:transport:wsgi]
-[drivers:proxy]
-storage = mongodb
-transport = wsgi
-
-[drivers:transport:wsgi]
 ;bind = 0.0.0.0
 ;port = 8888
 
@@ -41,14 +40,10 @@ transport = wsgi
 ;metadata_max_length = 65536
 ;content_max_length = 262144
 
-;[drivers:transport:zmq]
+;[queues:drivers:transport:zmq]
 ;port = 9999
 
-;[drivers:proxy:transport:wsgi]
+[queues:drivers:storage:mongodb]
-;bind = 0.0.0.0
-;port = 8889
-
-[drivers:storage:mongodb]
 uri = mongodb://db1.example.net,db2.example.net:2500/?replicaSet=test&ssl=true&w=majority
 database = marconi
 
@@ -72,11 +67,7 @@ database = marconi
 # at the same instant.
 ;max_retry_jitter = 0.005
 
-[drivers:proxy:storage:mongodb]
+[queues:limits:transport]
-uri = mongodb://localhost:27017
-database = marconi_proxy
-
-[limits:transport]
 # The maximum number of queue records per page when listing queues
 ;queue_paging_uplimit = 20
 
@@ -95,14 +86,9 @@ database = marconi_proxy
 ;metadata_size_uplimit = 65536
 ;message_size_uplimit = 262144
 
-[limits:storage]
+[queues:limits:storage]
 # The default number of queue records per page when listing queues
 ;default_queue_paging = 10
 
 # The default number of messages per page when listing or claiming messages
 ;default_message_paging = 10
-
-# caching mechanism
-[oslo_cache]
-cache_backend = memory
-;cache_prefix = my_prefix

marconi/common/config.py

@@ -101,7 +101,7 @@ def _init():
 
         return Obj(from_options=from_options)
 
-    def project(name=None):
+    def project(name=None, prog=None):
         """Access the global namespace.
 
         :param name: the name of the project
@@ -141,7 +141,7 @@ def _init():
             args = [] if args is None else args
 
             if filename is None:
-                conf(args=args, project=name, prog=name)
+                conf(args=args, project=name, prog=prog)
             else:
                 conf(args=args, default_config_files=[filename])
 

marconi/proxy/bootstrap.py

@@ -24,8 +24,8 @@ from marconi.openstack.common import log
 from marconi.proxy import transport  # NOQA
 
 
-PROJECT_CFG = config.project('marconi')
+PROJECT_CFG = config.project('marconi', 'marconi-proxy')
-CFG = config.namespace('drivers:proxy').from_options(
+CFG = config.namespace('proxy:drivers').from_options(
     transport='wsgi',
     storage='memory')
 

marconi/proxy/storage/mongodb/options.py

@@ -25,5 +25,6 @@ OPTIONS = {
     'database': 'marconi_proxy'
 }
 
-NAMESPACE = 'drivers:proxy:storage:mongodb'
+CFG = config.namespace('proxy:drivers:storage:mongodb').from_options(
-CFG = config.namespace(NAMESPACE).from_options(**OPTIONS)
+    **OPTIONS
+)

marconi/proxy/transport/wsgi/driver.py

@@ -45,11 +45,11 @@ OPTIONS = {
     'port': 8889
 }
 
-PROJECT_CFG = config.project('marconi')
+PROJECT_CFG = config.project('marconi', 'marconi-proxy')
 GLOBAL_CFG = PROJECT_CFG.from_options()
-WSGI_CFG = config.namespace(
+WSGI_CFG = config.namespace('proxy:drivers:transport:wsgi').from_options(
-    'drivers:proxy:transport:wsgi'
+    **OPTIONS
-).from_options(**OPTIONS)
+)
 
 LOG = logging.getLogger(__name__)
 

marconi/queues/bootstrap.py

@@ -22,8 +22,8 @@ from marconi.openstack.common import log
 from marconi.queues import transport  # NOQA
 
 
-PROJECT_CFG = config.project('marconi')
+PROJECT_CFG = config.project('marconi', 'marconi-queues')
-CFG = config.namespace('drivers').from_options(
+CFG = config.namespace('queues:drivers').from_options(
     transport='wsgi',
     storage='sqlite')
 

marconi/queues/storage/mongodb/claims.py

@@ -33,7 +33,7 @@ from marconi.queues.storage import exceptions
 from marconi.queues.storage.mongodb import utils
 
 LOG = logging.getLogger(__name__)
-CFG = config.namespace('limits:storage').from_options(
+CFG = config.namespace('queues:limits:storage').from_options(
     default_message_paging=10,
 )
 

marconi/queues/storage/mongodb/messages.py

@@ -36,7 +36,7 @@ from marconi.queues.storage.mongodb import options
 from marconi.queues.storage.mongodb import utils
 
 LOG = logging.getLogger(__name__)
-CFG = config.namespace('limits:storage').from_options(
+CFG = config.namespace('queues:limits:storage').from_options(
     default_message_paging=10,
 )
 

marconi/queues/storage/mongodb/options.py

@@ -48,4 +48,6 @@ OPTIONS = {
     'max_retry_jitter': 0.005,
 }
 
-CFG = config.namespace('drivers:storage:mongodb').from_options(**OPTIONS)
+CFG = config.namespace('queues:drivers:storage:mongodb').from_options(
+    **OPTIONS
+)

marconi/queues/storage/mongodb/queues.py

@@ -31,7 +31,7 @@ from marconi.queues.storage import exceptions
 from marconi.queues.storage.mongodb import utils
 
 LOG = logging.getLogger(__name__)
-CFG = config.namespace('limits:storage').from_options(
+CFG = config.namespace('queues:limits:storage').from_options(
     default_queue_paging=10,
 )
 

marconi/queues/storage/sqlite/claims.py

@@ -18,7 +18,7 @@ from marconi.queues.storage import base
 from marconi.queues.storage import exceptions
 from marconi.queues.storage.sqlite import utils
 
-CFG = config.namespace('limits:storage').from_options(
+CFG = config.namespace('queues:limits:storage').from_options(
     default_message_paging=10,
 )
 

marconi/queues/storage/sqlite/driver.py

@@ -23,7 +23,7 @@ from marconi.queues import storage
 from marconi.queues.storage.sqlite import controllers
 from marconi.queues.storage.sqlite import utils
 
-CFG = config.namespace('drivers:storage:sqlite').from_options(
+CFG = config.namespace('queues:drivers:storage:sqlite').from_options(
     database=':memory:')
 
 

marconi/queues/storage/sqlite/messages.py

@@ -19,7 +19,7 @@ from marconi.queues.storage import base
 from marconi.queues.storage import exceptions
 from marconi.queues.storage.sqlite import utils
 
-CFG = config.namespace('limits:storage').from_options(
+CFG = config.namespace('queues:limits:storage').from_options(
     default_message_paging=10,
 )
 

marconi/queues/storage/sqlite/queues.py

@@ -19,7 +19,7 @@ from marconi.queues.storage import base
 from marconi.queues.storage import exceptions
 from marconi.queues.storage.sqlite import utils
 
-CFG = config.namespace('limits:storage').from_options(
+CFG = config.namespace('queues:limits:storage').from_options(
     default_queue_paging=10,
 )
 

marconi/queues/transport/validation.py

@@ -30,7 +30,7 @@ OPTIONS = {
     'claim_grace_max': 43200,
 }
 
-CFG = config.namespace('limits:transport').from_options(**OPTIONS)
+CFG = config.namespace('queues:limits:transport').from_options(**OPTIONS)
 
 QUEUE_NAME_REGEX = re.compile('^[\w-]+$')
 

marconi/queues/transport/wsgi/claims.py

@@ -26,7 +26,7 @@ from marconi.queues.transport.wsgi import utils as wsgi_utils
 
 
 LOG = logging.getLogger(__name__)
-CFG = config.namespace('drivers:transport:wsgi').from_options(
+CFG = config.namespace('queues:drivers:transport:wsgi').from_options(
     metadata_max_length=64 * 1024
 )
 

marconi/queues/transport/wsgi/driver.py

@@ -33,9 +33,11 @@ OPTIONS = {
     'port': 8888
 }
 
-PROJECT_CFG = config.project('marconi')
+PROJECT_CFG = config.project('marconi', 'marconi-queues')
 GLOBAL_CFG = PROJECT_CFG.from_options()
-WSGI_CFG = config.namespace('drivers:transport:wsgi').from_options(**OPTIONS)
+WSGI_CFG = config.namespace('queues:drivers:transport:wsgi').from_options(
+    **OPTIONS
+)
 
 LOG = logging.getLogger(__name__)
 

marconi/queues/transport/wsgi/messages.py

@@ -25,7 +25,7 @@ from marconi.queues.transport.wsgi import utils as wsgi_utils
 
 
 LOG = logging.getLogger(__name__)
-CFG = config.namespace('drivers:transport:wsgi').from_options(
+CFG = config.namespace('queues:drivers:transport:wsgi').from_options(
     content_max_length=256 * 1024
 )
 

marconi/queues/transport/wsgi/metadata.py

@@ -26,7 +26,7 @@ from marconi.queues.transport.wsgi import utils as wsgi_utils
 
 
 LOG = logging.getLogger(__name__)
-CFG = config.namespace('drivers:transport:wsgi').from_options(
+CFG = config.namespace('queues:drivers:transport:wsgi').from_options(
     metadata_max_length=64 * 1024
 )
 

marconi/tests/functional/base.py

@@ -52,7 +52,7 @@ class FunctionalTestBase(testing.TestBase):
             self.server.start(self.conf_path(self.cfg.marconi.config))
 
         self.mconf = self.load_conf(self.cfg.marconi.config).conf
-        self.limits = self.mconf['limits:transport']
+        self.limits = self.mconf['queues:limits:transport']
 
         # NOTE(flaper87): Create client
         # for this test unit.

tests/etc/drivers_storage_invalid.conf

@@ -2,9 +2,9 @@
 debug = False
 verbose = False
 
-[drivers]
+[queues:drivers]
 transport = wsgi
 storage = invalid
 
-[drivers:transport:wsgi]
+[queues:drivers:transport:wsgi]
 port = 8888

tests/etc/drivers_transport_invalid.conf

@@ -2,9 +2,9 @@
 debug = False
 verbose = False
 
-[drivers]
+[queues:drivers]
 transport = invalid
 storage = sqlite
 
-[drivers:transport:wsgi]
+[queues:drivers:transport:wsgi]
 port = 8888

tests/etc/functional-marconi.conf

@@ -20,13 +20,13 @@ debug = True
 ;syslog_log_facility = LOG_LOCAL0
 
 
-[drivers]
+[queues:drivers]
 # Transport driver module (e.g., wsgi, zmq)
 transport = wsgi
 # Storage driver module (e.g., mongodb, sqlite)
 storage = sqlite
 
-[drivers:transport:wsgi]
+[queues:drivers:transport:wsgi]
 bind = 127.0.0.1
 port = 8888
 
@@ -35,10 +35,10 @@ port = 8888
 ;metadata_max_length = 65536
 ;content_max_length = 262144
 
-;[drivers:transport:zmq]
+;[queues:drivers:transport:zmq]
 ;port = 9999
 
-[limits:transport]
+[queues:limits:transport]
 # The maximum number of queue records per page when listing queues
 ;queue_paging_uplimit = 20
 # The maximum number of messages in a message posting, maximum

tests/etc/keystone_auth.conf

@@ -4,10 +4,10 @@ auth_strategy = keystone
 debug = False
 verbose = False
 
-[drivers]
+[queues:drivers]
 transport = wsgi
 storage = sqlite
 
-[drivers:transport:wsgi]
+[queues:drivers:transport:wsgi]
 bind = 0.0.0.0:8888
 workers = 20

tests/etc/wsgi_faulty.conf

@@ -2,9 +2,9 @@
 debug = False
 verbose = False
 
-[drivers]
+[queues:drivers]
 transport = wsgi
 storage = sqlite
 
-[drivers:transport:wsgi]
+[queues:drivers:transport:wsgi]
 port = 8888

tests/etc/wsgi_mongodb.conf

@@ -2,13 +2,13 @@
 debug = False
 verbose = False
 
-[drivers]
+[queues:drivers]
 transport = wsgi
 storage = mongodb
 
-[drivers:transport:wsgi]
+[queues:drivers:transport:wsgi]
 port = 8888
 
-[drivers:storage:mongodb]
+[queues:drivers:storage:mongodb]
 uri = mongodb://127.0.0.1:27017
 database = marconi_test

tests/etc/wsgi_proxy_mongodb.conf

@@ -2,13 +2,13 @@
 debug = False
 verbose = False
 
-[drivers]
+[proxy:drivers]
 transport = wsgi
 storage = mongodb
 
-[drivers:transport:wsgi]
+[proxy:drivers:transport:wsgi]
 port = 8888
 
-[drivers:proxy:storage:mongodb]
+[proxy:drivers:storage:mongodb]
 uri = mongodb://127.0.0.1:27017
 database = marconi_proxy_test

tests/etc/wsgi_sqlite.conf

@@ -2,11 +2,11 @@
 debug = False
 verbose = False
 
-[drivers]
+[queues:drivers]
 transport = wsgi
 storage = sqlite
 
-[drivers:transport:wsgi]
+[queues:drivers:transport:wsgi]
 bind = 0.0.0.0
 port = 8888
 workers = 20

tests/etc/wsgi_sqlite_default_limits.conf

@@ -1,7 +1,7 @@
-[drivers]
+[queues:drivers]
 transport = wsgi
 storage = sqlite
 
-[limits:storage]
+[queues:limits:storage]
 default_queue_paging = 1
 default_message_paging = 2

tests/etc/wsgi_sqlite_validation.conf

@@ -1,7 +1,7 @@
-[drivers]
+[queues:drivers]
 transport = wsgi
 storage = sqlite
 
-[limits:transport]
+[queues:limits:transport]
 metadata_size_uplimit = 64
 message_size_uplimit = 256

tests/unit/queues/transport/wsgi/test_claims.py

@@ -32,7 +32,7 @@ class ClaimsBaseTest(base.TestBase):
         super(ClaimsBaseTest, self).setUp()
 
         self.wsgi_cfg = config.namespace(
-            'drivers:transport:wsgi').from_options()
+            'queues:drivers:transport:wsgi').from_options()
 
         self.project_id = '480924'
         self.queue_path = '/v1/queues/fizbit'
@@ -237,7 +237,9 @@ class ClaimsMongoDBTests(ClaimsBaseTest):
 
         super(ClaimsMongoDBTests, self).setUp()
 
-        self.cfg = config.namespace('drivers:storage:mongodb').from_options()
+        self.cfg = config.namespace(
+            'queues:drivers:storage:mongodb'
+        ).from_options()
 
     def tearDown(self):
         storage = self.boot.storage

tests/unit/queues/transport/wsgi/test_messages.py

@@ -32,7 +32,7 @@ class MessagesBaseTest(base.TestBase):
         super(MessagesBaseTest, self).setUp()
 
         self.wsgi_cfg = config.namespace(
-            'drivers:transport:wsgi').from_options()
+            'queues:drivers:transport:wsgi').from_options()
 
         self.project_id = '7e55e1a7e'
         self.queue_path = '/v1/queues/fizbit'

tests/unit/queues/transport/wsgi/test_queue_lifecycle.py

@@ -33,7 +33,7 @@ class QueueLifecycleBaseTest(base.TestBase):
         super(QueueLifecycleBaseTest, self).setUp()
 
         self.wsgi_cfg = config.namespace(
-            'drivers:transport:wsgi').from_options()
+            'queues:drivers:transport:wsgi').from_options()
 
     def test_empty_project_id(self):
         path = '/v1/queues/gumshoe'
@@ -259,7 +259,9 @@ class QueueLifecycleMongoDBTests(QueueLifecycleBaseTest):
             self.skipTest('No MongoDB instance running')
         super(QueueLifecycleMongoDBTests, self).setUp()
 
-        self.cfg = config.namespace('drivers:storage:mongodb').from_options()
+        self.cfg = config.namespace(
+            'queues:drivers:storage:mongodb'
+        ).from_options()
 
     def tearDown(self):
         storage = self.boot.storage