vzstorage: add ploop volume format support

This patch adds support for a volume format, which can
be used for containers in Virtuozzo Server.

This format is different from qcow2 images, because it consists
of 2 or more files in a directory.

The first file is an xml description of a bundle. It includes
paths to base image and deltas, which usually stored in the same
directory. Here is a description of base/delta images format -
https://openvz.org/Ploop/format.

Format name is a little ambiguous, because ploop is not only an
image format name, but also a technology, which allows to use
images, stored in files as block devices (like loop devices or
qemu-nbd), but it supports snapshots and has nearly native
performance. qemu-img has limited support of this format - it
can only convert image files, the format is called 'parallels'
in qemu-img.

The main utility to manage such disks is 'ploop'. Here is a man
page - https://static.openvz.org/vz-man/man8/ploop.8.gz.html.
It can create images, make snapshots and create ploop devices,
based on images.

This patch series makes possible all volume operations except for
snapshotting of 'in-use' volumes.

DocImpact
Depends-On: Iebd20544102672d7b7ca820c9e9005833ec2580e
Change-Id: Id3c902c38192fddc02529afe5af7abc213c7b57c

cinder/tests/unit/test_vzstorage.py

@@ -284,7 +284,7 @@ class VZStorageTestCase(test.TestCase):
             self.snap, self.vol,
             self.vol['size'])
         drv._extend_volume.assert_called_once_with(
-            self.vol, self.vol['size'])
+            self.vol, self.vol['size'], 'raw')
         mock_convert_image.assert_called_once_with(
             self._FAKE_VOLUME_PATH, self._FAKE_VOLUME_PATH[:-1], 'raw')
 

cinder/volume/drivers/vzstorage.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import collections
 import errno
 import json
 import os
@@ -56,11 +57,67 @@ vzstorage_opts = [
                 help=('Mount options passed to the vzstorage client. '
                       'See section of the pstorage-mount man page '
                       'for details.')),
+    cfg.StrOpt('vzstorage_default_volume_format',
+               default='raw',
+               help=('Default format that will be used when creating volumes '
+                     'if no volume format is specified.')),
 ]
 
 CONF = cfg.CONF
 CONF.register_opts(vzstorage_opts)
 
+PLOOP_BASE_DELTA_NAME = 'root.hds'
+DISK_FORMAT_RAW = 'raw'
+DISK_FORMAT_QCOW2 = 'qcow2'
+DISK_FORMAT_PLOOP = 'parallels'
+
+
+class PloopDevice(object):
+    """Setup a ploop device for parallels image
+
+    This class is for mounting ploop devices using with statement:
+    with PloopDevice('/parallels/my-vm/harddisk.hdd') as dev_path:
+        # do something
+
+    :param path: A path to parallels harddisk dir
+    :param snapshot_id: Snapshot id to mount
+    :param execute: execute helper
+    """
+
+    def __init__(self, path, snapshot_id=None, read_only=True,
+                 execute=putils.execute):
+        self.path = path
+        self.snapshot_id = snapshot_id
+        self.read_only = read_only
+        self.execute = execute
+
+    def __enter__(self):
+        self.dd_path = os.path.join(self.path, 'DiskDescriptor.xml')
+        cmd = ['ploop', 'mount', self.dd_path]
+
+        if self.snapshot_id:
+            cmd.append('-u')
+            cmd.append(self.snapshot_id)
+
+        if self.read_only:
+            cmd.append('-r')
+
+        out, err = self.execute(*cmd, run_as_root=True)
+
+        m = re.search('dev=(\S+)', out)
+        if not m:
+            raise Exception('Invalid output from ploop mount: %s' % out)
+
+        self.ploop_dev = m.group(1)
+
+        return self.ploop_dev
+
+    def _umount(self):
+        self.execute('ploop', 'umount', self.dd_path, run_as_root=True)
+
+    def __exit__(self, type, value, traceback):
+        self._umount()
+
 
 class VZStorageDriver(remotefs_drv.RemoteFSSnapDriver):
     """Cinder driver for Virtuozzo Storage.
@@ -108,13 +165,9 @@ class VZStorageDriver(remotefs_drv.RemoteFSSnapDriver):
         """
         # Find active image
         active_file = self.get_active_image_from_info(volume)
-        active_file_path = os.path.join(self._local_volume_dir(volume),
-                                        active_file)
-        info = self._qemu_img_info(active_file_path, volume.name)
-        fmt = info.file_format
 
         data = {'export': volume.provider_location,
-                'format': fmt,
+                'format': self.get_volume_format(volume),
                 'name': active_file,
                 }
 
@@ -226,25 +279,92 @@ class VZStorageDriver(remotefs_drv.RemoteFSSnapDriver):
 
         return True
 
+    def choose_volume_format(self, volume):
+        vol_type = volume.volume_type
+        if vol_type:
+            extra_specs = vol_type.extra_specs or {}
+        else:
+            extra_specs = {}
+
+        extra_specs.update(volume.metadata or {})
+
+        return (extra_specs.get('volume_format') or
+                self.configuration.vzstorage_default_volume_format)
+
+    def get_volume_format(self, volume):
+        info_path = self._local_path_volume_info(volume)
+        snap_info = self._read_info_file(info_path)
+        return snap_info['volume_format']
+
+    def _create_ploop(self, volume_path, volume_size):
+        os.mkdir(volume_path)
+        try:
+            self._execute('ploop', 'init', '-s', '%sG' % volume_size,
+                          os.path.join(volume_path, PLOOP_BASE_DELTA_NAME),
+                          run_as_root=True)
+        except putils.ProcessExecutionError:
+            os.rmdir(volume_path)
+            raise
+
+    def _do_create_volume(self, volume):
+        """Create a volume on given smbfs_share.
+
+        :param volume: volume reference
+        """
+        volume_format = self.choose_volume_format(volume)
+        volume_path = self.local_path(volume)
+        volume_size = volume.size
+
+        LOG.debug("Creating new volume at %s.", volume_path)
+
+        if os.path.exists(volume_path):
+            msg = _('File already exists at %s.') % volume_path
+            LOG.error(msg)
+            raise exception.InvalidVolume(reason=msg)
+
+        if volume_format == 'parallels':
+            self._create_ploop(volume_path, volume_size)
+        elif volume_format == 'qcow2':
+            self._create_qcow2_file(volume_path, volume_size)
+        elif self.configuration.vzstorage_sparsed_volumes:
+            self._create_sparsed_file(volume_path, volume_size)
+        else:
+            self._create_regular_file(volume_path, volume_size)
+
+        info_path = self._local_path_volume_info(volume)
+        snap_info = {'volume_format': volume_format,
+                     'active': 'volume-%s' % volume.id}
+        self._write_info_file(info_path, snap_info)
+
+    def _delete(self, path):
+        self._execute('rm', '-rf', path, run_as_root=True)
+
     @remotefs_drv.locked_volume_id_operation
     def extend_volume(self, volume, size_gb):
         LOG.info(_LI('Extending volume %s.'), volume.id)
-        self._extend_volume(volume, size_gb)
+        volume_format = self.get_volume_format(volume)
+        self._extend_volume(volume, size_gb, volume_format)
 
-    def _extend_volume(self, volume, size_gb):
+    def _extend_volume(self, volume, size_gb, volume_format):
         volume_path = self.local_path(volume)
 
         self._check_extend_volume_support(volume, size_gb)
         LOG.info(_LI('Resizing file to %sG...'), size_gb)
 
-        self._do_extend_volume(volume_path, size_gb)
+        self._do_extend_volume(volume_path, size_gb, volume_format)
 
-    def _do_extend_volume(self, volume_path, size_gb):
+    def _do_extend_volume(self, volume_path, size_gb, volume_format):
-        image_utils.resize_image(volume_path, size_gb)
+        if volume_format == "parallels":
-
+            self._execute('ploop', 'grow', '-s',
-        if not self._is_file_size_equal(volume_path, size_gb):
+                          '%dG' % size_gb,
-            raise exception.ExtendVolumeError(
+                          os.path.join(volume_path, 'DiskDescriptor.xml'),
-                reason='Resizing image file failed.')
+                          run_as_root=True)
+            volume_path = os.path.join(volume_path, PLOOP_BASE_DELTA_NAME)
+        else:
+            image_utils.resize_image(volume_path, size_gb)
+            if not self._is_file_size_equal(volume_path, size_gb):
+                raise exception.ExtendVolumeError(
+                    reason='Resizing image file failed.')
 
     def _check_extend_volume_support(self, volume, size_gb):
         volume_path = self.local_path(volume)
@@ -270,6 +390,30 @@ class VZStorageDriver(remotefs_drv.RemoteFSSnapDriver):
         virt_size = data.virtual_size / units.Gi
         return virt_size == size
 
+    def _recreate_ploop_desc(self, image_dir, image_file):
+        self._delete(os.path.join(image_dir, 'DiskDescriptor.xml'))
+
+        self._execute('ploop', 'restore-descriptor', image_dir, image_file)
+
+    def copy_image_to_volume(self, context, volume, image_service, image_id):
+        """Fetch the image from image_service and write it to the volume."""
+        volume_format = self.get_volume_format(volume)
+        image_path = self.local_path(volume)
+        if volume_format == DISK_FORMAT_PLOOP:
+            image_path = os.path.join(image_path, PLOOP_BASE_DELTA_NAME)
+
+        image_utils.fetch_to_volume_format(
+            context, image_service, image_id,
+            image_path, volume_format,
+            self.configuration.volume_dd_blocksize)
+
+        if volume_format == DISK_FORMAT_PLOOP:
+            self._recreate_ploop_desc(self.local_path(volume), image_path)
+
+        self._do_extend_volume(self.local_path(volume),
+                               volume.size,
+                               volume_format)
+
     def _copy_volume_from_snapshot(self, snapshot, volume, volume_size):
         """Copy data from snapshot to destination volume.
 
@@ -277,34 +421,46 @@ class VZStorageDriver(remotefs_drv.RemoteFSSnapDriver):
         qcow2.
         """
 
-        LOG.debug("_copy_volume_from_snapshot: snapshot: %(snap)s, "
-                  "volume: %(vol)s, volume_size: %(size)s.",
-                  {'snap': snapshot.id,
-                   'vol': volume.id,
-                   'size': volume_size,
-                   })
-
         info_path = self._local_path_volume_info(snapshot.volume)
         snap_info = self._read_info_file(info_path)
         vol_dir = self._local_volume_dir(snapshot.volume)
-        out_format = "raw"
+        out_format = self.choose_volume_format(volume)
+        volume_format = self.get_volume_format(snapshot.volume)
+        volume_path = self.local_path(volume)
 
-        forward_file = snap_info[snapshot.id]
+        if volume_format in (DISK_FORMAT_QCOW2, DISK_FORMAT_RAW):
-        forward_path = os.path.join(vol_dir, forward_file)
+            forward_file = snap_info[snapshot.id]
+            forward_path = os.path.join(vol_dir, forward_file)
 
-        # Find the file which backs this file, which represents the point
+            # Find the file which backs this file, which represents the point
-        # when this snapshot was created.
+            # when this snapshot was created.
-        img_info = self._qemu_img_info(forward_path,
+            img_info = self._qemu_img_info(forward_path,
-                                       snapshot.volume.name)
+                                           snapshot.volume.name)
-        path_to_snap_img = os.path.join(vol_dir, img_info.backing_file)
+            path_to_snap_img = os.path.join(vol_dir, img_info.backing_file)
 
-        LOG.debug("_copy_volume_from_snapshot: will copy "
+            LOG.debug("_copy_volume_from_snapshot: will copy "
-                  "from snapshot at %s.", path_to_snap_img)
+                      "from snapshot at %s.", path_to_snap_img)
 
-        image_utils.convert_image(path_to_snap_img,
+            image_utils.convert_image(path_to_snap_img,
-                                  self.local_path(volume),
+                                      volume_path,
-                                  out_format)
+                                      out_format)
-        self._extend_volume(volume, volume_size)
+        elif volume_format == DISK_FORMAT_PLOOP:
+            with PloopDevice(self.local_path(snapshot.volume),
+                             snapshot.id,
+                             execute=self._execute) as dev:
+                image_utils.convert_image(dev, volume_path, out_format)
+        else:
+            msg = _("Unsupported volume format %s") % volume_format
+            raise exception.InvalidVolume(msg)
+
+        if out_format == DISK_FORMAT_PLOOP:
+            img_path = os.path.join(volume_path, 'root.hds')
+            os.rename(volume_path, volume_path + '.tmp')
+            os.mkdir(volume_path)
+            os.rename(volume_path + '.tmp', img_path)
+            self._execute('ploop', 'restore-descriptor', volume_path, img_path)
+
+        self._extend_volume(volume, volume_size, out_format)
 
     @remotefs_drv.locked_volume_id_operation
     def delete_volume(self, volume):
@@ -313,7 +469,8 @@ class VZStorageDriver(remotefs_drv.RemoteFSSnapDriver):
             msg = (_('Volume %s does not have provider_location '
                      'specified, skipping.') % volume.name)
             LOG.error(msg)
-            raise exception.VzStorageException(msg)
+            return
+#            raise exception.VzStorageException(msg)
 
         self._ensure_share_mounted(volume.provider_location)
         volume_dir = self._local_volume_dir(volume)
@@ -327,3 +484,120 @@ class VZStorageDriver(remotefs_drv.RemoteFSSnapDriver):
 
         info_path = self._local_path_volume_info(volume)
         self._delete(info_path)
+
+    def _get_desc_path(self, volume):
+        return os.path.join(self.local_path(volume), 'DiskDescriptor.xml')
+
+    def _create_snapshot_ploop(self, snapshot):
+        status = snapshot.volume.status
+        if status != 'available':
+            msg = (_('Volume status must be available for '
+                     'snapshot %(id)s. (is %(status)s)') %
+                   {'id': snapshot.id, 'status': status})
+            raise exception.InvalidVolume(msg)
+
+        info_path = self._local_path_volume_info(snapshot.volume)
+        snap_info = self._read_info_file(info_path)
+        self._execute('ploop', 'snapshot', '-u', '{%s}' % snapshot.id,
+                      self._get_desc_path(snapshot.volume),
+                      run_as_root=True)
+        snap_file = os.path.join('volume-%s' % snapshot.volume.id, snapshot.id)
+        snap_info[snapshot.id] = snap_file
+        self._write_info_file(info_path, snap_info)
+
+    def _delete_snapshot_ploop(self, snapshot):
+        status = snapshot.volume.status
+        if status != 'available':
+            msg = (_('Volume status must be available for '
+                     'snapshot %(id)s. (is %(status)s)') %
+                   {'id': snapshot.id, 'status': status})
+            raise exception.InvalidVolume(msg)
+
+        info_path = self._local_path_volume_info(snapshot.volume)
+        snap_info = self._read_info_file(info_path)
+        self._execute('ploop', 'snapshot-delete', '-u', '{%s}' % snapshot.id,
+                      self._get_desc_path(snapshot.volume),
+                      run_as_root=True)
+        snap_info.pop(snapshot.id)
+        self._write_info_file(info_path, snap_info)
+
+    def create_snapshot(self, snapshot):
+        volume_format = self.get_volume_format(snapshot.volume)
+        if volume_format == 'parallels':
+            self._create_snapshot_ploop(snapshot)
+        else:
+            super(VZStorageDriver, self)._create_snapshot(snapshot)
+
+    def delete_snapshot(self, snapshot):
+        volume_format = self.get_volume_format(snapshot.volume)
+        if volume_format == 'parallels':
+            self._delete_snapshot_ploop(snapshot)
+        else:
+            super(VZStorageDriver, self)._delete_snapshot(snapshot)
+
+    def _copy_volume_to_image(self, context, volume, image_service,
+                              image_meta):
+        """Copy the volume to the specified image."""
+
+        volume_format = self.get_volume_format(volume)
+        if volume_format == DISK_FORMAT_PLOOP:
+            with PloopDevice(self.local_path(volume),
+                             execute=self._execute) as dev:
+                image_utils.upload_volume(context,
+                                          image_service,
+                                          image_meta,
+                                          dev,
+                                          volume_format='raw')
+        else:
+            super(VZStorageDriver, self)._copy_volume_to_image(context, volume,
+                                                               image_service,
+                                                               image_meta)
+
+    def _create_cloned_volume(self, volume, src_vref):
+        LOG.info(_LI('Cloning volume %(src)s to volume %(dst)s'),
+                 {'src': src_vref.id,
+                  'dst': volume.id})
+
+        if src_vref.status != 'available':
+            msg = _("Volume status must be 'available'.")
+            raise exception.InvalidVolume(msg)
+
+        volume_name = CONF.volume_name_template % volume.id
+
+        # Create fake snapshot object
+        snap_attrs = ['volume_name', 'size', 'volume_size', 'name',
+                      'volume_id', 'id', 'volume']
+        Snapshot = collections.namedtuple('Snapshot', snap_attrs)
+
+        temp_snapshot = Snapshot(id=src_vref.id,
+                                 volume_name=volume_name,
+                                 size=src_vref.size,
+                                 volume_size=src_vref.size,
+                                 name='clone-snap-%s' % src_vref.id,
+                                 volume_id=src_vref.id,
+                                 volume=volume)
+
+        self._create_snapshot_ploop(temp_snapshot)
+        try:
+            volume.provider_location = src_vref.provider_location
+            info_path = self._local_path_volume_info(volume)
+            snap_info = {'volume_format': DISK_FORMAT_PLOOP,
+                         'active': 'volume-%s' % volume.id}
+            self._write_info_file(info_path, snap_info)
+            self._copy_volume_from_snapshot(temp_snapshot,
+                                            volume,
+                                            volume.size)
+
+        finally:
+            self.delete_snapshot(temp_snapshot)
+
+        return {'provider_location': src_vref.provider_location}
+
+    @remotefs_drv.locked_volume_id_operation
+    def create_cloned_volume(self, vol, src_vref):
+        """Creates a clone of the specified volume."""
+        volume_format = self.get_volume_format(src_vref)
+        if volume_format == 'parallels':
+            self._create_cloned_volume(vol, src_vref)
+        else:
+            super(VZStorageDriver, self)._create_cloned_volume(vol, src_vref)

etc/cinder/rootwrap.d/volume.filters

@@ -202,6 +202,7 @@ vgc-cluster: CommandFilter, vgc-cluster, root
 # cinder/volume/drivers/vzstorage.py
 pstorage-mount: CommandFilter, pstorage-mount, root
 pstorage: CommandFilter, pstorage, root
+ploop: CommandFilter, ploop, root
 
 # initiator/connector.py:
 drv_cfg: CommandFilter, /opt/emc/scaleio/sdc/bin/drv_cfg, root, /opt/emc/scaleio/sdc/bin/drv_cfg, --query_guid