Adds a Cache for Volumes Created from Snapshots with Quobyte

This is not related to the Cinder image cache.

In order to speed up the creation of multiple volumes from a single
snapshot this change adds a cache of volumes created by merging a
snapshots backing chain to the Quobyte driver.
This behaviour can be activated via a new config option
'quobyte_volume_from_snapshot_cache'.
Instead of merging a snapshots backing chain into a new volume each time
a volume is created from this snapshot, the new implementation merges the
backing chain into a volume in the new volume cache. New volumes to be
created from that snapshot are then copied from the cached volume and
no longer require the merging process.
Merging happens only for the first time when the cached volume copy is
created from a specific snapshot. Subsequent creations of volumes from
this snapshot are simply copied from the cache which requires no costly
backing chain merge.

Partial-Bug: #1715078

Change-Id: I2142b1c0a0cc2c4f85794416e702a326d3406b9d

cinder/tests/unit/volume/drivers/test_quobyte.py

@@ -18,6 +18,7 @@
 import errno
 import os
 import psutil
+import shutil
 import six
 import traceback
 
@@ -62,6 +63,18 @@ class QuobyteDriverTestCase(test.TestCase):
     SNAP_UUID = 'bacadaca-baca-daca-baca-dacadacadaca'
     SNAP_UUID_2 = 'bebedede-bebe-dede-bebe-dedebebedede'
 
+    def _get_fake_snapshot(self, src_volume):
+        snapshot = fake_snapshot.fake_snapshot_obj(
+            self.context,
+            volume_name=src_volume.name,
+            display_name='clone-snap-%s' % src_volume.id,
+            size=src_volume.size,
+            volume_size=src_volume.size,
+            volume_id=src_volume.id,
+            id=self.SNAP_UUID)
+        snapshot.volume = src_volume
+        return snapshot
+
     def setUp(self):
         super(QuobyteDriverTestCase, self).setUp()
 
@@ -76,6 +89,7 @@ class QuobyteDriverTestCase(test.TestCase):
             self.TEST_MNT_POINT_BASE
         self._configuration.nas_secure_file_operations = "auto"
         self._configuration.nas_secure_file_permissions = "auto"
+        self._configuration.quobyte_volume_from_snapshot_cache = False
 
         self._driver =\
             quobyte.QuobyteDriver(configuration=self._configuration,
@@ -118,6 +132,62 @@ class QuobyteDriverTestCase(test.TestCase):
                 'fallocate', '-l', '%sG' % test_size, tmp_path,
                 run_as_root=self._driver._execute_as_root)
 
+    @mock.patch.object(os, "makedirs")
+    @mock.patch.object(os.path, "join", return_value="dummy_path")
+    @mock.patch.object(os, "access", return_value=True)
+    def test__ensure_volume_cache_ok(self, os_access_mock, os_join_mock,
+                                     os_makedirs_mock):
+        tmp_path = "/some/random/path"
+
+        self._driver._ensure_volume_from_snap_cache(tmp_path)
+
+        calls = [mock.call("dummy_path", os.F_OK),
+                 mock.call("dummy_path", os.R_OK),
+                 mock.call("dummy_path", os.W_OK),
+                 mock.call("dummy_path", os.X_OK)]
+        os_access_mock.assert_has_calls(calls)
+        os_join_mock.assert_called_once_with(
+            tmp_path, self._driver.QUOBYTE_VOLUME_SNAP_CACHE_DIR_NAME)
+        self.assertFalse(os_makedirs_mock.called)
+
+    @mock.patch.object(os, "makedirs")
+    @mock.patch.object(os.path, "join", return_value="dummy_path")
+    @mock.patch.object(os, "access", return_value=True)
+    def test__ensure_volume_cache_create(self, os_access_mock, os_join_mock,
+                                         os_makedirs_mock):
+        tmp_path = "/some/random/path"
+        os_access_mock.side_effect = [False, True, True, True]
+
+        self._driver._ensure_volume_from_snap_cache(tmp_path)
+
+        calls = [mock.call("dummy_path", os.F_OK),
+                 mock.call("dummy_path", os.R_OK),
+                 mock.call("dummy_path", os.W_OK),
+                 mock.call("dummy_path", os.X_OK)]
+        os_access_mock.assert_has_calls(calls)
+        os_join_mock.assert_called_once_with(
+            tmp_path, self._driver.QUOBYTE_VOLUME_SNAP_CACHE_DIR_NAME)
+        os_makedirs_mock.assert_called_once_with("dummy_path")
+
+    @mock.patch.object(os, "makedirs")
+    @mock.patch.object(os.path, "join", return_value="dummy_path")
+    @mock.patch.object(os, "access", return_value=True)
+    def test__ensure_volume_cache_error(self, os_access_mock, os_join_mock,
+                                        os_makedirs_mock):
+        tmp_path = "/some/random/path"
+        os_access_mock.side_effect = [True, False, False, False]
+
+        self.assertRaises(
+            exception.VolumeDriverException,
+            self._driver._ensure_volume_from_snap_cache, tmp_path)
+
+        calls = [mock.call("dummy_path", os.F_OK),
+                 mock.call("dummy_path", os.R_OK)]
+        os_access_mock.assert_has_calls(calls)
+        os_join_mock.assert_called_once_with(
+            tmp_path, self._driver.QUOBYTE_VOLUME_SNAP_CACHE_DIR_NAME)
+        self.assertFalse(os_makedirs_mock.called)
+
     def test_local_path(self):
         """local_path common use case."""
         drv = self._driver
@@ -166,8 +236,8 @@ class QuobyteDriverTestCase(test.TestCase):
                                         self.TEST_MNT_POINT)
             mock_validate.assert_called_once_with(self.TEST_MNT_POINT)
 
-    def test_mount_quobyte_should_suppress_and_log_already_mounted_error(self):
+    def test_mount_quobyte_should_suppress_already_mounted_error(self):
-        """test_mount_quobyte_should_suppress_and_log_already_mounted_error
+        """test_mount_quobyte_should_suppress_already_mounted_error
 
            Based on /proc/mount, the file system is not mounted yet. However,
            mount.quobyte returns with an 'already mounted' error. This is
@@ -175,12 +245,13 @@ class QuobyteDriverTestCase(test.TestCase):
            successful.
 
            Because _mount_quobyte gets called with ensure=True, the error will
-           be suppressed and logged instead.
+           be suppressed instead.
         """
         with mock.patch.object(self._driver, '_execute') as mock_execute, \
                 mock.patch('cinder.volume.drivers.quobyte.QuobyteDriver'
                            '.read_proc_mount') as mock_open, \
-                mock.patch('cinder.volume.drivers.quobyte.LOG') as mock_LOG:
+                mock.patch('cinder.volume.drivers.quobyte.QuobyteDriver'
+                           '._validate_volume') as mock_validate:
             # Content of /proc/mount (empty).
             mock_open.return_value = six.StringIO()
             mock_execute.side_effect = [None, putils.ProcessExecutionError(
@@ -196,14 +267,12 @@ class QuobyteDriverTestCase(test.TestCase):
                 self.TEST_MNT_POINT, run_as_root=False)
             mock_execute.assert_has_calls([mkdir_call, mount_call],
                                           any_order=False)
-
+            mock_validate.assert_called_once_with(self.TEST_MNT_POINT)
-            mock_LOG.warning.assert_called_once_with('%s is already mounted',
-                                                     self.TEST_QUOBYTE_VOLUME)
 
     def test_mount_quobyte_should_reraise_already_mounted_error(self):
         """test_mount_quobyte_should_reraise_already_mounted_error
 
-        Like test_mount_quobyte_should_suppress_and_log_already_mounted_error
+        Like test_mount_quobyte_should_suppress_already_mounted_error
         but with ensure=False.
         """
         with mock.patch.object(self._driver, '_execute') as mock_execute, \
@@ -228,6 +297,68 @@ class QuobyteDriverTestCase(test.TestCase):
             mock_execute.assert_has_calls([mkdir_call, mount_call],
                                           any_order=False)
 
+    @mock.patch.object(image_utils, "qemu_img_info")
+    def test_optimize_volume_not(self, iu_qii_mock):
+        drv = self._driver
+        vol = self._simple_volume()
+        vol.size = 3
+        img_data = mock.Mock()
+        img_data.disk_size = 3 * units.Gi
+        iu_qii_mock.return_value = img_data
+        drv._execute = mock.Mock()
+        drv._create_regular_file = mock.Mock()
+        drv.local_path = mock.Mock(return_value="/some/path")
+
+        drv.optimize_volume(vol)
+
+        iu_qii_mock.assert_called_once_with("/some/path",
+                                            run_as_root=drv._execute_as_root)
+        self.assertFalse(drv._execute.called)
+        self.assertFalse(drv._create_regular_file.called)
+
+    @mock.patch.object(image_utils, "qemu_img_info")
+    def test_optimize_volume_sparse(self, iu_qii_mock):
+        drv = self._driver
+        vol = self._simple_volume()
+        vol.size = 3
+        img_data = mock.Mock()
+        img_data.disk_size = 2 * units.Gi
+        iu_qii_mock.return_value = img_data
+        drv._execute = mock.Mock()
+        drv._create_regular_file = mock.Mock()
+        drv.local_path = mock.Mock(return_value="/some/path")
+
+        drv.optimize_volume(vol)
+
+        iu_qii_mock.assert_called_once_with(drv.local_path(),
+                                            run_as_root=drv._execute_as_root)
+        drv._execute.assert_called_once_with(
+            'truncate', '-s', '%sG' % vol.size, drv.local_path(),
+            run_as_root=drv._execute_as_root)
+        self.assertFalse(drv._create_regular_file.called)
+
+    @mock.patch.object(image_utils, "qemu_img_info")
+    def test_optimize_volume_regular(self, iu_qii_mock):
+        drv = self._driver
+        drv.configuration.quobyte_qcow2_volumes = False
+        drv.configuration.quobyte_sparsed_volumes = False
+        vol = self._simple_volume()
+        vol.size = 3
+        img_data = mock.Mock()
+        img_data.disk_size = 2 * units.Gi
+        iu_qii_mock.return_value = img_data
+        drv._execute = mock.Mock()
+        drv._create_regular_file = mock.Mock()
+        drv.local_path = mock.Mock(return_value="/some/path")
+
+        drv.optimize_volume(vol)
+
+        iu_qii_mock.assert_called_once_with(drv.local_path(),
+                                            run_as_root=drv._execute_as_root)
+        self.assertFalse(drv._execute.called)
+        drv._create_regular_file.assert_called_once_with(drv.local_path(),
+                                                         vol.size)
+
     def test_get_hash_str(self):
         """_get_hash_str should calculation correct value."""
         drv = self._driver
@@ -643,15 +774,7 @@ class QuobyteDriverTestCase(test.TestCase):
         dest_vol_path = os.path.join(vol_dir, dest_volume['name'])
         info_path = os.path.join(vol_dir, src_volume['name']) + '.info'
 
-        snapshot = fake_snapshot.fake_snapshot_obj(
+        snapshot = self._get_fake_snapshot(src_volume)
-            self.context,
-            volume_name=src_volume.name,
-            display_name='clone-snap-%s' % src_volume.id,
-            size=src_volume.size,
-            volume_size=src_volume.size,
-            volume_id=src_volume.id,
-            id=self.SNAP_UUID)
-        snapshot.volume = src_volume
 
         snap_file = dest_volume['name'] + '.' + snapshot['id']
         snap_path = os.path.join(vol_dir, snap_file)
@@ -672,9 +795,8 @@ class QuobyteDriverTestCase(test.TestCase):
                                         {'active': snap_file,
                                          snapshot['id']: snap_file})
         image_utils.qemu_img_info = mock.Mock(return_value=img_info)
-        drv._set_rw_permissions_for_all = mock.Mock()
+        drv._set_rw_permissions = mock.Mock()
-        drv._find_share = mock.Mock()
+        drv.optimize_volume = mock.Mock()
-        drv._find_share.return_value = "/some/arbitrary/path"
 
         drv._copy_volume_from_snapshot(snapshot, dest_volume, size)
 
@@ -687,7 +809,124 @@ class QuobyteDriverTestCase(test.TestCase):
                                  dest_vol_path,
                                  'raw',
                                  run_as_root=self._driver._execute_as_root))
-        drv._set_rw_permissions_for_all.assert_called_once_with(dest_vol_path)
+        drv._set_rw_permissions.assert_called_once_with(dest_vol_path)
+        drv.optimize_volume.assert_called_once_with(dest_volume)
+
+    @mock.patch.object(os, "access", return_value=True)
+    def test_copy_volume_from_snapshot_cached(self, os_ac_mock):
+        drv = self._driver
+        drv.configuration.quobyte_volume_from_snapshot_cache = True
+
+        # lots of test vars to be prepared at first
+        dest_volume = self._simple_volume(
+            id='c1073000-0000-0000-0000-0000000c1073')
+        src_volume = self._simple_volume()
+
+        vol_dir = os.path.join(self.TEST_MNT_POINT_BASE,
+                               drv._get_hash_str(self.TEST_QUOBYTE_VOLUME))
+        dest_vol_path = os.path.join(vol_dir, dest_volume['name'])
+        info_path = os.path.join(vol_dir, src_volume['name']) + '.info'
+
+        snapshot = self._get_fake_snapshot(src_volume)
+
+        snap_file = dest_volume['name'] + '.' + snapshot['id']
+        snap_path = os.path.join(vol_dir, snap_file)
+        cache_path = os.path.join(vol_dir,
+                                  drv.QUOBYTE_VOLUME_SNAP_CACHE_DIR_NAME,
+                                  snapshot['id'])
+
+        size = dest_volume['size']
+
+        qemu_img_output = """image: %s
+        file format: raw
+        virtual size: 1.0G (1073741824 bytes)
+        disk size: 173K
+        backing file: %s
+        """ % (snap_file, src_volume['name'])
+        img_info = imageutils.QemuImgInfo(qemu_img_output)
+
+        # mocking and testing starts here
+        image_utils.convert_image = mock.Mock()
+        drv._read_info_file = mock.Mock(return_value=
+                                        {'active': snap_file,
+                                         snapshot['id']: snap_file})
+        image_utils.qemu_img_info = mock.Mock(return_value=img_info)
+        drv._set_rw_permissions = mock.Mock()
+        shutil.copyfile = mock.Mock()
+        drv.optimize_volume = mock.Mock()
+
+        drv._copy_volume_from_snapshot(snapshot, dest_volume, size)
+
+        drv._read_info_file.assert_called_once_with(info_path)
+        image_utils.qemu_img_info.assert_called_once_with(snap_path,
+                                                          force_share=False,
+                                                          run_as_root=False)
+        self.assertFalse(image_utils.convert_image.called,
+                         ("_convert_image was called but should not have been")
+                         )
+        os_ac_mock.assert_called_once_with(
+            drv._local_volume_from_snap_cache_path(snapshot), os.F_OK)
+        shutil.copyfile.assert_called_once_with(cache_path, dest_vol_path)
+        drv._set_rw_permissions.assert_called_once_with(dest_vol_path)
+        drv.optimize_volume.assert_called_once_with(dest_volume)
+
+    def test_copy_volume_from_snapshot_not_cached(self):
+        drv = self._driver
+        drv.configuration.quobyte_volume_from_snapshot_cache = True
+
+        # lots of test vars to be prepared at first
+        dest_volume = self._simple_volume(
+            id='c1073000-0000-0000-0000-0000000c1073')
+        src_volume = self._simple_volume()
+
+        vol_dir = os.path.join(self.TEST_MNT_POINT_BASE,
+                               drv._get_hash_str(self.TEST_QUOBYTE_VOLUME))
+        src_vol_path = os.path.join(vol_dir, src_volume['name'])
+        dest_vol_path = os.path.join(vol_dir, dest_volume['name'])
+        info_path = os.path.join(vol_dir, src_volume['name']) + '.info'
+
+        snapshot = self._get_fake_snapshot(src_volume)
+
+        snap_file = dest_volume['name'] + '.' + snapshot['id']
+        snap_path = os.path.join(vol_dir, snap_file)
+        cache_path = os.path.join(vol_dir,
+                                  drv.QUOBYTE_VOLUME_SNAP_CACHE_DIR_NAME,
+                                  snapshot['id'])
+
+        size = dest_volume['size']
+
+        qemu_img_output = """image: %s
+        file format: raw
+        virtual size: 1.0G (1073741824 bytes)
+        disk size: 173K
+        backing file: %s
+        """ % (snap_file, src_volume['name'])
+        img_info = imageutils.QemuImgInfo(qemu_img_output)
+
+        # mocking and testing starts here
+        image_utils.convert_image = mock.Mock()
+        drv._read_info_file = mock.Mock(return_value=
+                                        {'active': snap_file,
+                                         snapshot['id']: snap_file})
+        image_utils.qemu_img_info = mock.Mock(return_value=img_info)
+        drv._set_rw_permissions = mock.Mock()
+        shutil.copyfile = mock.Mock()
+        drv.optimize_volume = mock.Mock()
+
+        drv._copy_volume_from_snapshot(snapshot, dest_volume, size)
+
+        drv._read_info_file.assert_called_once_with(info_path)
+        image_utils.qemu_img_info.assert_called_once_with(snap_path,
+                                                          force_share=False,
+                                                          run_as_root=False)
+        (image_utils.convert_image.
+         assert_called_once_with(
+             src_vol_path,
+             drv._local_volume_from_snap_cache_path(snapshot), 'raw',
+             run_as_root=self._driver._execute_as_root))
+        shutil.copyfile.assert_called_once_with(cache_path, dest_vol_path)
+        drv._set_rw_permissions.assert_called_once_with(dest_vol_path)
+        drv.optimize_volume.assert_called_once_with(dest_volume)
 
     def test_create_volume_from_snapshot_status_not_available(self):
         """Expect an error when the snapshot's status is not 'available'."""

cinder/volume/drivers/quobyte.py

@@ -17,13 +17,16 @@
 import errno
 import os
 import psutil
+import shutil
 
 from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import fileutils
+from oslo_utils import units
 
 from cinder import compute
+from cinder import coordination
 from cinder import exception
 from cinder.i18n import _
 from cinder.image import image_utils
@@ -32,7 +35,7 @@ from cinder import utils
 from cinder.volume import configuration
 from cinder.volume.drivers import remotefs as remotefs_drv
 
-VERSION = '1.1.7'
+VERSION = '1.1.8'
 
 LOG = logging.getLogger(__name__)
 
@@ -54,6 +57,11 @@ volume_opts = [
                default='$state_path/mnt',
                help=('Base dir containing the mount point'
                      ' for the Quobyte volume.')),
+    cfg.BoolOpt('quobyte_volume_from_snapshot_cache',
+                default=False,
+                help=('Create a cache of volumes from merged snapshots to '
+                      'speed up creation of multiple volumes from a single '
+                      'snapshot.'))
 ]
 
 CONF = cfg.CONF
@@ -88,6 +96,7 @@ class QuobyteDriver(remotefs_drv.RemoteFSSnapDriverDistributed):
         1.1.5 - Enables extension of volumes with snapshots
         1.1.6 - Optimizes volume creation
         1.1.7 - Support fuse subtype based Quobyte mount validation
+        1.1.8 - Adds optional snapshot merge caching
 
     """
 
@@ -99,6 +108,8 @@ class QuobyteDriver(remotefs_drv.RemoteFSSnapDriverDistributed):
     # ThirdPartySystems wiki page
     CI_WIKI_NAME = "Quobyte_CI"
 
+    QUOBYTE_VOLUME_SNAP_CACHE_DIR_NAME = "volume_from_snapshot_cache"
+
     def __init__(self, execute=processutils.execute, *args, **kwargs):
         super(QuobyteDriver, self).__init__(*args, **kwargs)
         self.configuration.append_config_values(volume_opts)
@@ -111,6 +122,32 @@ class QuobyteDriver(remotefs_drv.RemoteFSSnapDriverDistributed):
         self._execute('fallocate', '-l', '%sG' % size,
                       path, run_as_root=self._execute_as_root)
 
+    def _ensure_volume_from_snap_cache(self, mount_path):
+        """This expects the Quobyte volume to be mounted & available"""
+        cache_path = os.path.join(mount_path,
+                                  self.QUOBYTE_VOLUME_SNAP_CACHE_DIR_NAME)
+        if not os.access(cache_path, os.F_OK):
+            LOG.info("Volume from snapshot cache directory does not exist, "
+                     "creating the directory %(volcache)s",
+                     {'volcache': cache_path})
+            os.makedirs(cache_path)
+        if not (os.access(cache_path, os.R_OK)
+                and os.access(cache_path, os.W_OK)
+                and os.access(cache_path, os.X_OK)):
+            msg = _("Insufficient permissions for Quobyte volume from "
+                    "snapshot cache directory at %(cpath)s. Please update "
+                    "permissions.") % {'cpath': cache_path}
+            raise exception.VolumeDriverException(msg)
+        LOG.debug("Quobyte volume from snapshot cache directory validated ok")
+
+    def _local_volume_from_snap_cache_path(self, snapshot):
+        path_to_disk = os.path.join(
+            self._local_volume_dir(snapshot.volume),
+            self.QUOBYTE_VOLUME_SNAP_CACHE_DIR_NAME,
+            snapshot.id)
+
+        return path_to_disk
+
     def do_setup(self, context):
         """Any initialization the volume driver does while starting."""
         super(QuobyteDriver, self).do_setup(context)
@@ -138,6 +175,32 @@ class QuobyteDriver(remotefs_drv.RemoteFSSnapDriverDistributed):
             else:
                 raise
 
+    def optimize_volume(self, volume):
+        """Optimizes a volume for Quobyte
+
+        This optimization is normally done during creation but volumes created
+        from e.g. snapshots require additional grooming.
+
+        :param volume: volume reference
+        """
+        volume_path = self.local_path(volume)
+        volume_size = volume.size
+        data = image_utils.qemu_img_info(self.local_path(volume),
+                                         run_as_root=self._execute_as_root)
+        if data.disk_size >= (volume_size * units.Gi):
+            LOG.debug("Optimization of volume %(volpath)s is not required, "
+                      "skipping this step.", {'volpath': volume_path})
+            return
+
+        LOG.debug("Optimizing volume %(optpath)s", {'optpath': volume_path})
+
+        if (self.configuration.quobyte_qcow2_volumes or
+                self.configuration.quobyte_sparsed_volumes):
+                self._execute('truncate', '-s', '%sG' % volume_size,
+                              volume_path, run_as_root=self._execute_as_root)
+        else:
+            self._create_regular_file(volume_path, volume_size)
+
     def set_nas_security_options(self, is_new_cinder_install):
         self._execute_as_root = False
 
@@ -222,18 +285,20 @@ class QuobyteDriver(remotefs_drv.RemoteFSSnapDriverDistributed):
     def create_volume_from_snapshot(self, volume, snapshot):
         return self._create_volume_from_snapshot(volume, snapshot)
 
+    @coordination.synchronized('{self.driver_prefix}-{snapshot.volume.id}')
     def _copy_volume_from_snapshot(self, snapshot, volume, volume_size):
         """Copy data from snapshot to destination volume.
 
         This is done with a qemu-img convert to raw/qcow2 from the snapshot
-        qcow2.
+        qcow2. If the quobyte_volume_from_snapshot_cache is active the result
+        is copied into the cache and all volumes created from this
+        snapshot id are directly copied from the cache.
         """
 
         LOG.debug("snapshot: %(snap)s, volume: %(vol)s, ",
                   {'snap': snapshot.id,
                    'vol': volume.id,
                    'size': volume_size})
-
         info_path = self._local_path_volume_info(snapshot.volume)
         snap_info = self._read_info_file(info_path)
         vol_path = self._local_volume_dir(snapshot.volume)
@@ -248,6 +313,7 @@ class QuobyteDriver(remotefs_drv.RemoteFSSnapDriverDistributed):
         path_to_snap_img = os.path.join(vol_path, img_info.backing_file)
 
         path_to_new_vol = self._local_path_volume(volume)
+        path_to_cached_vol = self._local_volume_from_snap_cache_path(snapshot)
 
         LOG.debug("will copy from snapshot at %s", path_to_snap_img)
 
@@ -256,12 +322,27 @@ class QuobyteDriver(remotefs_drv.RemoteFSSnapDriverDistributed):
         else:
             out_format = 'raw'
 
-        image_utils.convert_image(path_to_snap_img,
+        if not self.configuration.quobyte_volume_from_snapshot_cache:
-                                  path_to_new_vol,
+            LOG.debug("Creating direct copy from snapshot")
-                                  out_format,
+            image_utils.convert_image(path_to_snap_img,
-                                  run_as_root=self._execute_as_root)
+                                      path_to_new_vol,
-
+                                      out_format,
-        self._set_rw_permissions_for_all(path_to_new_vol)
+                                      run_as_root=self._execute_as_root)
+        else:
+            # create the volume via volume cache
+            if not os.access(path_to_cached_vol, os.F_OK):
+                LOG.debug("Caching volume %(volpath)s from snapshot.",
+                          {'volpath': path_to_cached_vol})
+                image_utils.convert_image(path_to_snap_img,
+                                          path_to_cached_vol,
+                                          out_format,
+                                          run_as_root=self._execute_as_root)
+            # Copy volume from cache
+            LOG.debug("Copying volume %(volpath)s from cache",
+                      {'volpath': path_to_new_vol})
+            shutil.copyfile(path_to_cached_vol, path_to_new_vol)
+        self._set_rw_permissions(path_to_new_vol)
+        self.optimize_volume(volume)
 
     @utils.synchronized('quobyte', external=False)
     def delete_volume(self, volume):
@@ -299,6 +380,9 @@ class QuobyteDriver(remotefs_drv.RemoteFSSnapDriverDistributed):
     def delete_snapshot(self, snapshot):
         """Apply locking to the delete snapshot operation."""
         self._delete_snapshot(snapshot)
+        if self.configuration.quobyte_volume_from_snapshot_cache:
+            fileutils.delete_if_exists(
+                self._local_volume_from_snap_cache_path(snapshot))
 
     @utils.synchronized('quobyte', external=False)
     def initialize_connection(self, volume, connector):
@@ -495,11 +579,14 @@ class QuobyteDriver(remotefs_drv.RemoteFSSnapDriverDistributed):
             except processutils.ProcessExecutionError as exc:
                 if ensure and 'already mounted' in exc.stderr:
                     LOG.warning("%s is already mounted", quobyte_volume)
+                    mounted = True
                 else:
                     raise
 
         if mounted:
             self._validate_volume(mount_path)
+            if self.configuration.quobyte_volume_from_snapshot_cache:
+                self._ensure_volume_from_snap_cache(mount_path)
 
     def _validate_volume(self, mount_path):
         """Runs a number of tests on the expect Quobyte mount"""

releasenotes/notes/quobyte_vol-snap-cache-baf607f14d916ec7.yaml

@@ -0,0 +1,9 @@
+---
+
+fixes:
+  - |
+    Added a new optional cache of volumes generated from snapshots for the
+    Quobyte backend. Enabling this cache speeds up creation of multiple
+    volumes from a single snapshot at the cost of a slight increase in
+    creation time for the first volume generated for this given snapshot.
+    The ``quobyte_volume_from_snapshot_cache`` option is off by default.