Try to use a default pool when creating a queue

We recently enabled default pools creation, which will use the default
`message_storage` if present. This patch extends that to queue's
creation where we try to register a queue in the catalogue but we fail
if no pools were found.

With this patch, the failure will happen after checking if there's a
default pool that can be used.

In addition to the above, this commit adds a new option
`enable_virtual_pool` that makes default pools registration optional.
This is to avoid breaking backwards compatibility on the deployments
already running Zaqar.

DocImpact

Closes-bug: #1462008
Closes-bug: #1463110
Change-Id: Ieb8d2671066fc909f422bd74fd20070bc0f9cf11

devstack/plugin.sh

@@ -127,6 +127,7 @@ function configure_zaqar {
 
     if [ "$ZAQAR_BACKEND" = 'mongodb' ] ; then
         iniset $ZAQAR_CONF DEFAULT pooling True
+        iniset $ZAQAR_CONF 'pooling:catalog' enable_virtual_pool True
         iniset $ZAQAR_CONF  drivers message_store mongodb
         iniset $ZAQAR_CONF 'drivers:message_store:mongodb' uri mongodb://localhost:27017/zaqar
         iniset $ZAQAR_CONF 'drivers:message_store:mongodb' database zaqar

tests/unit/storage/test_pool_catalog.py

@@ -67,9 +67,14 @@ class PoolCatalogTest(testing.TestBase):
         storage = self.catalog.lookup(self.queue, self.project)
         self.assertIsInstance(storage._storage, mongodb.DataDriver)
 
-    def test_lookup_returns_none_if_queue_not_mapped(self):
+    def test_lookup_returns_default_or_none_if_queue_not_mapped(self):
+        # Return default
         self.assertIsNone(self.catalog.lookup('not', 'mapped'))
 
+        self.config(message_store='faulty', group='drivers')
+        self.config(enable_virtual_pool=True, group='pooling:catalog')
+        self.assertIsNotNone(self.catalog.lookup('not', 'mapped'))
+
     def test_lookup_returns_none_if_entry_deregistered(self):
         self.catalog.deregister(self.queue, self.project)
         self.assertIsNone(self.catalog.lookup(self.queue, self.project))

zaqar/storage/pooling.py

@@ -20,6 +20,7 @@ from oslo_config import cfg
 from oslo_log import log
 
 from zaqar.common import decorators
+from zaqar.common import errors as cerrors
 from zaqar.common.storage import select
 from zaqar import storage
 from zaqar.storage import errors
@@ -31,6 +32,9 @@ LOG = log.getLogger(__name__)
 _CATALOG_OPTIONS = (
     cfg.StrOpt('storage', default='sqlalchemy',
                help='Catalog storage driver.'),
+    cfg.BoolOpt('enable_virtual_pool', default=False,
+                help=('If enabled, the message_store will be used '
+                      'as the storage for the virtual pool.')),
 )
 
 _CATALOG_GROUP = 'pooling:catalog'
@@ -489,7 +493,13 @@ class Catalog(object):
                 pool = select.weighted(pools)
                 pool = pool and pool['name'] or None
 
-                if not pool:
+                if not pool and self.lookup(queue, project) is None:
+                    # NOTE(flaper87): We used to raise NoPoolFound in this
+                    # case but we've decided to support automatic pool
+                    # creation. Note that we're now returning and the queue
+                    # is not being registered in the catalogue. This is done
+                    # on purpose since no pool exists and the "dummy" pool
+                    # doesn't exist in the storage
                     raise errors.NoPoolFound()
 
             self._catalogue_ctrl.insert(project, queue, pool)
@@ -584,16 +594,31 @@ class Catalog(object):
         except errors.QueueNotMapped as ex:
             LOG.debug(ex)
 
+            if not self._catalog_conf.enable_virtual_pool:
+                return None
+
             conf_section = ('drivers:message_store:%s' %
                             self._conf.drivers.message_store)
 
-            if conf_section not in self._conf:
+            try:
+                # NOTE(flaper87): Try to load the driver to check
+                # whether it can be used as the default store for
+                # the default pool.
+                utils.load_storage_driver(self._conf, self._cache,
+                                          control_driver=self.control)
+            except cerrors.InvalidDriver:
                 # NOTE(kgriffs): Return `None`, rather than letting the
                 # exception bubble up, so that the higher layer doesn't
                 # have to duplicate the try..except..log code all over
                 # the place.
                 return None
 
+            if conf_section not in self._conf:
+                # NOTE(flaper87): If there's no config section for this storage
+                # skip the pool registration entirely since we won't know how
+                # to connect to it.
+                return None
+
             # NOTE(flaper87): This assumes the storage driver type is the
             # same as the management.
             pool_conf = {'uri': self._conf[conf_section].uri,

zaqar/tests/faulty_storage.py

@@ -13,10 +13,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from oslo_config import cfg
+
 from zaqar import storage
 
+_CONFIG_GROUP = 'drivers:message_store:faulty'
+
 
 class DataDriver(storage.DataDriverBase):
+
+    _DRIVER_OPTIONS = [(_CONFIG_GROUP,
+                        [cfg.StrOpt('uri', default='faulty://')])]
+
     def __init__(self, conf, cache, control_driver):
         super(DataDriver, self).__init__(conf, cache, control_driver)