Change Marconi's codebase structure

This patch re-structures Marconi's codebase in a way that it'll be
possible to add other services such as marconi-proxy and
marconi-notification as standalone services within Marconi.

Implements blueprint change-marconi-structure

Change-Id: Id6061f04ec6e6959143fb0f5905bb8799fd86edd

marconi/__init__.py

@@ -22,8 +22,8 @@ try:
 except NameError:
     import gettext
     gettext.install('marconi', unicode=1)
-    import marconi.bootstrap
+    import marconi.queues.bootstrap
-    Bootstrap = marconi.bootstrap.Bootstrap
+    Bootstrap = marconi.queues.bootstrap.Bootstrap
 
 import marconi.version
 

marconi/queues/bootstrap.py

@@ -19,7 +19,7 @@ from marconi.common import config
 from marconi.common import decorators
 from marconi.common import exceptions
 from marconi.openstack.common import log
-from marconi import transport  # NOQA
+from marconi.queues import transport  # NOQA
 
 
 PROJECT_CFG = config.project('marconi')

marconi/queues/storage/__init__.py

@@ -1,7 +1,7 @@
 """Marconi Storage Drivers"""
 
-from marconi.storage import base
+from marconi.queues.storage import base
-from marconi.storage import exceptions  # NOQA
+from marconi.queues.storage import exceptions  # NOQA
 
 
 # Hoist classes into package namespace

marconi/queues/storage/mongodb/__init__.py

@@ -1,6 +1,6 @@
 """MongoDB Storage Driver for Marconi"""
 
-from marconi.storage.mongodb import driver
+from marconi.queues.storage.mongodb import driver
 
 # Hoist classes into package namespace
 Driver = driver.Driver

marconi/queues/storage/mongodb/claims.py

@@ -26,9 +26,9 @@ from bson import objectid
 from marconi.common import config
 import marconi.openstack.common.log as logging
 from marconi.openstack.common import timeutils
-from marconi import storage
+from marconi.queues import storage
-from marconi.storage import exceptions
+from marconi.queues.storage import exceptions
-from marconi.storage.mongodb import utils
+from marconi.queues.storage.mongodb import utils
 
 LOG = logging.getLogger(__name__)
 CFG = config.namespace('limits:storage').from_options(

marconi/queues/storage/mongodb/controllers.py

@@ -22,9 +22,9 @@ Field Mappings:
     updated and documented in each controller class.
 """
 
-from marconi.storage.mongodb import claims
+from marconi.queues.storage.mongodb import claims
-from marconi.storage.mongodb import messages
+from marconi.queues.storage.mongodb import messages
-from marconi.storage.mongodb import queues
+from marconi.queues.storage.mongodb import queues
 
 ClaimController = claims.ClaimController
 MessageController = messages.MessageController

marconi/queues/storage/mongodb/driver.py

@@ -19,9 +19,9 @@ import pymongo
 import pymongo.errors
 
 from marconi.openstack.common import log as logging
-from marconi import storage
+from marconi.queues import storage
-from marconi.storage.mongodb import controllers
+from marconi.queues.storage.mongodb import controllers
-from marconi.storage.mongodb import options
+from marconi.queues.storage.mongodb import options
 
 LOG = logging.getLogger(__name__)
 

marconi/queues/storage/mongodb/messages.py

@@ -29,10 +29,10 @@ import pymongo.read_preferences
 from marconi.common import config
 import marconi.openstack.common.log as logging
 from marconi.openstack.common import timeutils
-from marconi import storage
+from marconi.queues import storage
-from marconi.storage import exceptions
+from marconi.queues.storage import exceptions
-from marconi.storage.mongodb import options
+from marconi.queues.storage.mongodb import options
-from marconi.storage.mongodb import utils
+from marconi.queues.storage.mongodb import utils
 
 LOG = logging.getLogger(__name__)
 CFG = config.namespace('limits:storage').from_options(

marconi/queues/storage/mongodb/queues.py

@@ -26,9 +26,9 @@ import pymongo.errors
 from marconi.common import config
 import marconi.openstack.common.log as logging
 from marconi.openstack.common import timeutils
-from marconi import storage
+from marconi.queues import storage
-from marconi.storage import exceptions
+from marconi.queues.storage import exceptions
-from marconi.storage.mongodb import utils
+from marconi.queues.storage.mongodb import utils
 
 LOG = logging.getLogger(__name__)
 CFG = config.namespace('limits:storage').from_options(

marconi/queues/storage/mongodb/utils.py

@@ -27,7 +27,7 @@ from pymongo import errors
 from marconi.common import exceptions
 import marconi.openstack.common.log as logging
 from marconi.openstack.common import timeutils
-from marconi.storage import exceptions as storage_exceptions
+from marconi.queues.storage import exceptions as storage_exceptions
 
 
 DUP_MARKER_REGEX = re.compile(r'\$queue_marker.*?:\s(\d+)')

marconi/queues/storage/sqlite/__init__.py

@@ -4,7 +4,7 @@ In-memory reference Storage Driver for Marconi.
 Useful for automated testing and for prototyping storage driver concepts.
 """
 
-from marconi.storage.sqlite import driver
+from marconi.queues.storage.sqlite import driver
 
 # Hoist classes into package namespace
 Driver = driver.Driver

marconi/queues/storage/sqlite/claims.py

@@ -14,9 +14,9 @@
 # limitations under the License.
 
 from marconi.common import config
-from marconi.storage import base
+from marconi.queues.storage import base
-from marconi.storage import exceptions
+from marconi.queues.storage import exceptions
-from marconi.storage.sqlite import utils
+from marconi.queues.storage.sqlite import utils
 
 CFG = config.namespace('limits:storage').from_options(
     default_message_paging=10,

marconi/queues/storage/sqlite/controllers.py

@@ -16,9 +16,9 @@
 
 """Exports SQLite driver controllers."""
 
-from marconi.storage.sqlite import claims
+from marconi.queues.storage.sqlite import claims
-from marconi.storage.sqlite import messages
+from marconi.queues.storage.sqlite import messages
-from marconi.storage.sqlite import queues
+from marconi.queues.storage.sqlite import queues
 
 ClaimController = claims.ClaimController
 MessageController = messages.MessageController

marconi/queues/storage/sqlite/driver.py

@@ -19,9 +19,9 @@ import sqlite3
 import msgpack
 
 from marconi.common import config
-from marconi import storage
+from marconi.queues import storage
-from marconi.storage.sqlite import controllers
+from marconi.queues.storage.sqlite import controllers
-from marconi.storage.sqlite import utils
+from marconi.queues.storage.sqlite import utils
 
 CFG = config.namespace('drivers:storage:sqlite').from_options(
     database=':memory:')

marconi/queues/storage/sqlite/messages.py

@@ -15,9 +15,9 @@
 
 from marconi.common import config
 from marconi.openstack.common import timeutils
-from marconi.storage import base
+from marconi.queues.storage import base
-from marconi.storage import exceptions
+from marconi.queues.storage import exceptions
-from marconi.storage.sqlite import utils
+from marconi.queues.storage.sqlite import utils
 
 CFG = config.namespace('limits:storage').from_options(
     default_message_paging=10,

marconi/queues/storage/sqlite/queues.py

@@ -15,9 +15,9 @@
 # limitations under the License.
 
 from marconi.common import config
-from marconi.storage import base
+from marconi.queues.storage import base
-from marconi.storage import exceptions
+from marconi.queues.storage import exceptions
-from marconi.storage.sqlite import utils
+from marconi.queues.storage.sqlite import utils
 
 CFG = config.namespace('limits:storage').from_options(
     default_queue_paging=10,

marconi/queues/storage/sqlite/utils.py

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from marconi.storage import exceptions
+from marconi.queues.storage import exceptions
 
 UNIX_EPOCH_AS_JULIAN_SEC = 2440587.5 * 86400.0
 

marconi/queues/transport/__init__.py

@@ -1,7 +1,7 @@
 """Marconi Transport Drivers"""
 
 from marconi.common import config
-from marconi.transport import base
+from marconi.queues.transport import base
 
 OPTIONS = {
     'auth_strategy': ""

marconi/queues/transport/wsgi/__init__.py

@@ -0,0 +1,11 @@
+"""WSGI Transport Driver"""
+
+from marconi.queues.transport.wsgi import claims  # NOQA
+from marconi.queues.transport.wsgi import driver
+from marconi.queues.transport.wsgi import messages  # NOQA
+from marconi.queues.transport.wsgi import queues  # NOQA
+from marconi.queues.transport.wsgi import stats  # NOQA
+
+
+# Hoist into package namespace
+Driver = driver.Driver

marconi/queues/transport/wsgi/app.py

@@ -26,6 +26,6 @@ no common way to specify / pass configuration files
 to the WSGI app when it is called from other apps.
 """
 
-from marconi import bootstrap
+from marconi.queues import bootstrap
 
 app = bootstrap.Bootstrap().transport.app

marconi/queues/transport/wsgi/claims.py

@@ -18,11 +18,11 @@ import falcon
 from marconi.common import config
 from marconi.common import exceptions as input_exceptions
 import marconi.openstack.common.log as logging
-from marconi.storage import exceptions as storage_exceptions
+from marconi.queues.storage import exceptions as storage_exceptions
-from marconi.transport import utils
+from marconi.queues.transport import utils
-from marconi.transport import validation as validate
+from marconi.queues.transport import validation as validate
-from marconi.transport.wsgi import exceptions as wsgi_exceptions
+from marconi.queues.transport.wsgi import exceptions as wsgi_exceptions
-from marconi.transport.wsgi import utils as wsgi_utils
+from marconi.queues.transport.wsgi import utils as wsgi_utils
 
 
 LOG = logging.getLogger(__name__)

marconi/queues/transport/wsgi/driver.py

@@ -18,15 +18,15 @@ from wsgiref import simple_server
 
 from marconi.common import config
 import marconi.openstack.common.log as logging
-from marconi import transport
+from marconi.queues import transport
-from marconi.transport import auth
+from marconi.queues.transport import auth
-from marconi.transport.wsgi import claims
+from marconi.queues.transport.wsgi import claims
-from marconi.transport.wsgi import health
+from marconi.queues.transport.wsgi import health
-from marconi.transport.wsgi import messages
+from marconi.queues.transport.wsgi import messages
-from marconi.transport.wsgi import metadata
+from marconi.queues.transport.wsgi import metadata
-from marconi.transport.wsgi import queues
+from marconi.queues.transport.wsgi import queues
-from marconi.transport.wsgi import stats
+from marconi.queues.transport.wsgi import stats
-from marconi.transport.wsgi import v1
+from marconi.queues.transport.wsgi import v1
 
 OPTIONS = {
     'bind': '0.0.0.0',

marconi/queues/transport/wsgi/messages.py

@@ -17,11 +17,11 @@ import falcon
 from marconi.common import config
 from marconi.common import exceptions as input_exceptions
 import marconi.openstack.common.log as logging
-from marconi.storage import exceptions as storage_exceptions
+from marconi.queues.storage import exceptions as storage_exceptions
-from marconi.transport import utils
+from marconi.queues.transport import utils
-from marconi.transport import validation as validate
+from marconi.queues.transport import validation as validate
-from marconi.transport.wsgi import exceptions as wsgi_exceptions
+from marconi.queues.transport.wsgi import exceptions as wsgi_exceptions
-from marconi.transport.wsgi import utils as wsgi_utils
+from marconi.queues.transport.wsgi import utils as wsgi_utils
 
 
 LOG = logging.getLogger(__name__)

marconi/queues/transport/wsgi/metadata.py

@@ -18,11 +18,11 @@ import falcon
 from marconi.common import config
 from marconi.common import exceptions as input_exceptions
 import marconi.openstack.common.log as logging
-from marconi.storage import exceptions as storage_exceptions
+from marconi.queues.storage import exceptions as storage_exceptions
-from marconi.transport import utils
+from marconi.queues.transport import utils
-from marconi.transport import validation as validate
+from marconi.queues.transport import validation as validate
-from marconi.transport.wsgi import exceptions as wsgi_exceptions
+from marconi.queues.transport.wsgi import exceptions as wsgi_exceptions
-from marconi.transport.wsgi import utils as wsgi_utils
+from marconi.queues.transport.wsgi import utils as wsgi_utils
 
 
 LOG = logging.getLogger(__name__)

marconi/queues/transport/wsgi/queues.py

@@ -17,9 +17,9 @@ import falcon
 
 from marconi.common import exceptions as input_exceptions
 import marconi.openstack.common.log as logging
-from marconi.transport import utils
+from marconi.queues.transport import utils
-from marconi.transport import validation as validate
+from marconi.queues.transport import validation as validate
-from marconi.transport.wsgi import exceptions as wsgi_exceptions
+from marconi.queues.transport.wsgi import exceptions as wsgi_exceptions
 
 
 LOG = logging.getLogger(__name__)

marconi/queues/transport/wsgi/stats.py

@@ -16,9 +16,9 @@
 import falcon
 
 import marconi.openstack.common.log as logging
-from marconi.storage import exceptions as storage_exceptions
+from marconi.queues.storage import exceptions as storage_exceptions
-from marconi.transport import utils
+from marconi.queues.transport import utils
-from marconi.transport.wsgi import exceptions as wsgi_exceptions
+from marconi.queues.transport.wsgi import exceptions as wsgi_exceptions
 
 
 LOG = logging.getLogger(__name__)

marconi/queues/transport/wsgi/utils.py

@@ -16,8 +16,8 @@
 
 import marconi.openstack.common.log as logging
 
-from marconi.transport import utils
+from marconi.queues.transport import utils
-from marconi.transport.wsgi import exceptions
+from marconi.queues.transport.wsgi import exceptions
 
 
 JSONObject = dict

marconi/tests/faulty_storage.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 
-from marconi import storage
+from marconi.queues import storage
 
 
 class Driver(storage.DriverBase):

marconi/tests/functional/base.py

@@ -17,15 +17,15 @@
 import abc
 import multiprocessing
 
-from marconi import bootstrap
+from marconi.queues import bootstrap
+# NOTE(flaper87): This is necessary to register,
+# wsgi configs and won't be permanent. It'll be
+# refactored as part of the work for this blueprint
+from marconi.queues.transport import wsgi  # noqa
 from marconi import tests as testing
 from marconi.tests.functional import config
 from marconi.tests.functional import helpers
 from marconi.tests.functional import http
-# NOTE(flaper87): This is necessary to register,
-# wsgi configs and won't be permanent. It'll be
-# refactored as part of the work for this blueprint
-from marconi.transport import wsgi  # noqa
 
 
 class FunctionalTestBase(testing.TestBase):

marconi/transport/wsgi/__init__.py

@@ -1,11 +0,0 @@
-"""WSGI Transport Driver"""
-
-from marconi.transport.wsgi import claims  # NOQA
-from marconi.transport.wsgi import driver
-from marconi.transport.wsgi import messages  # NOQA
-from marconi.transport.wsgi import queues  # NOQA
-from marconi.transport.wsgi import stats  # NOQA
-
-
-# Hoist into package namespace
-Driver = driver.Driver

setup.cfg

@@ -31,11 +31,11 @@ console_scripts =
     marconi-server = marconi.cmd.server:run
 
 marconi.storage =
-    sqlite = marconi.storage.sqlite.driver:Driver
+    sqlite = marconi.queues.storage.sqlite.driver:Driver
-    mongodb = marconi.storage.mongodb.driver:Driver
+    mongodb = marconi.queues.storage.mongodb.driver:Driver
 
 marconi.transport =
-    wsgi = marconi.transport.wsgi.driver:Driver
+    wsgi = marconi.queues.transport.wsgi.driver:Driver
 
 marconi.common.cache.backends =
     memory = marconi.common.cache._backends.memory:MemoryBackend

tests/unit/storage/base.py

@@ -18,8 +18,8 @@ import time
 from testtools import matchers
 
 from marconi.openstack.common import timeutils
-from marconi import storage
+from marconi.queues import storage
-from marconi.storage import exceptions
+from marconi.queues.storage import exceptions
 from marconi import tests as testing
 
 

tests/unit/storage/test_impl_mongodb.py

@@ -22,11 +22,11 @@ import pymongo.errors
 from testtools import matchers
 
 from marconi.common import exceptions
-from marconi import storage
+from marconi.queues import storage
-from marconi.storage import mongodb
+from marconi.queues.storage import mongodb
-from marconi.storage.mongodb import controllers
+from marconi.queues.storage.mongodb import controllers
-from marconi.storage.mongodb import options as mongodb_options
+from marconi.queues.storage.mongodb import options as mongodb_options
-from marconi.storage.mongodb import utils
+from marconi.queues.storage.mongodb import utils
 from marconi import tests as testing
 
 import base  # noqa  # noqa

tests/unit/storage/test_impl_sqlite.py

@@ -13,9 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from marconi import storage
+from marconi.queues import storage
-from marconi.storage import sqlite
+from marconi.queues.storage import sqlite
-from marconi.storage.sqlite import controllers
+from marconi.queues.storage.sqlite import controllers
 
 import base  # noqa
 

tests/unit/test_bootstrap.py

@@ -15,11 +15,11 @@
 
 from oslo.config import cfg
 
-import marconi
 from marconi.common import exceptions
-from marconi.storage import sqlite
+import marconi.queues
+from marconi.queues.storage import sqlite
+from marconi.queues.transport import wsgi
 from marconi.tests import base
-from marconi.transport import wsgi
 
 
 class TestBootstrap(base.TestBase):

tests/unit/transport/test_auth.py

@@ -17,8 +17,8 @@
 from oslo.config import cfg
 
 from marconi.common import config
+from marconi.queues.transport import auth
 from marconi import tests as testing
-from marconi.transport import auth
 
 
 class TestTransportAuth(testing.TestBase):

tests/unit/transport/wsgi/base.py

@@ -16,7 +16,7 @@
 
 from falcon import testing as ftest
 
-import marconi
+import marconi.queues
 from marconi import tests as testing
 from marconi.tests import faulty_storage
 

tests/unit/transport/wsgi/test_utils.py

@@ -20,7 +20,7 @@ import falcon
 import json
 import testtools
 
-from marconi.transport.wsgi import utils
+from marconi.queues.transport.wsgi import utils
 
 
 class TestWSGIutils(testtools.TestCase):