From b71a594db6d06d33c4e61fa62f04e3700c74062b Mon Sep 17 00:00:00 2001 From: Dmitry Guryanov Date: Thu, 4 Feb 2016 22:36:03 +0300 Subject: [PATCH] libvirt: add nova volume driver for vzstorage Driver for vzstorage cinder volumes. The driver mounts vzstorage cluster and uses images, located on this mount as volumes. Snapshots of "in-use" volumes support: https://review.openstack.org/#/c/276465/ Cinder part has been merged, here are the bluprint and review: https://blueprints.launchpad.net/cinder/+spec/virtuozzo-cloud-storage-support https://review.openstack.org/#/c/188869/ Blueprint: libvirt-vzstorage-volume-support Change-Id: I6732fff3a5c40859781a017ef05046513685167f Co-Authored-By: Evgeny Antyshev --- etc/nova/rootwrap.d/compute.filters | 3 + nova/conf/libvirt.py | 140 +++++++++++++++++- .../virt/libvirt/volume/test_vzstorage.py | 109 ++++++++++++++ nova/virt/libvirt/driver.py | 2 + nova/virt/libvirt/volume/vzstorage.py | 127 ++++++++++++++++ 5 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 nova/tests/unit/virt/libvirt/volume/test_vzstorage.py create mode 100644 nova/virt/libvirt/volume/vzstorage.py diff --git a/etc/nova/rootwrap.d/compute.filters b/etc/nova/rootwrap.d/compute.filters index 1428c950cb2e..00766318383b 100644 --- a/etc/nova/rootwrap.d/compute.filters +++ b/etc/nova/rootwrap.d/compute.filters @@ -253,3 +253,6 @@ xend: CommandFilter, xend, root # nova/virt/libvirt/utils.py: touch: CommandFilter, touch, root + +# nova/virt/libvirt/volume/vzstorage.py +pstorage-mount: CommandFilter, pstorage-mount, root diff --git a/nova/conf/libvirt.py b/nova/conf/libvirt.py index 1dfd4b239222..0c50e0ab0245 100644 --- a/nova/conf/libvirt.py +++ b/nova/conf/libvirt.py @@ -413,6 +413,143 @@ libvirt_remotefs_opts = [ 'removing files on the remote host.'), ] +libvirt_volume_vzstorage_opts = [ + cfg.StrOpt('vzstorage_mount_point_base', + default=paths.state_path_def('mnt'), + help=""" +Directory where the Virtuozzo Storage clusters are mounted on the compute node. + +This option defines non-standard mountpoint for Vzstorage cluster. + +* Services that use this: + + ``nova-compute`` + +* Related options: + + vzstorage_mount_* group of parameters +""" + ), + cfg.StrOpt('vzstorage_mount_user', + default='stack', + help=""" +Mount owner user name. + +This option defines the owner user of Vzstorage cluster mountpoint. + +* Services that use this: + + ``nova-compute`` + +* Related options: + + vzstorage_mount_* group of parameters +""" + ), + cfg.StrOpt('vzstorage_mount_group', + default='qemu', + help=""" +Mount owner group name. + +This option defines the owner group of Vzstorage cluster mountpoint. + +* Services that use this: + + ``nova-compute`` + +* Related options: + + vzstorage_mount_* group of parameters +""" + ), + cfg.StrOpt('vzstorage_mount_perms', + default='0770', + help=""" +Mount access mode. + +This option defines the access bits of Vzstorage cluster mountpoint, +in the format similar to one of chmod(1) utility, like this: 0770. +It consists of one to four digits ranging from 0 to 7, with missing +lead digits assumed to be 0's. + +* Services that use this: + + ``nova-compute`` + +* Related options: + + vzstorage_mount_* group of parameters +""" + ), + cfg.StrOpt('vzstorage_log_path', + default='/var/log/pstorage/%(cluster_name)s/nova.log.gz', + help=""" +Path to vzstorage client log. + +This option defines the log of cluster operations, +it should include "%(cluster_name)s" template to separate +logs from multiple shares. + +* Services that use this: + + ``nova-compute`` + +* Related options: + + vzstorage_mount_opts may include more detailed logging options. +""" + ), + cfg.StrOpt('vzstorage_cache_path', + default=None, + help=""" +Path to the SSD cache file. + +You can attach an SSD drive to a client and configure the drive to store +a local cache of frequently accessed data. By having a local cache on a +client's SSD drive, you can increase the overall cluster performance by +up to 10 and more times. +WARNING! There is a lot of SSD models which are not server grade and +may loose arbitrary set of data changes on power loss. +Such SSDs should not be used in Vstorage and are dangerous as may lead +to data corruptions and inconsistencies. Please consult with the manual +on which SSD models are known to be safe or verify it using +vstorage-hwflush-check(1) utility. + +This option defines the path which should include "%(cluster_name)s" +template to separate caches from multiple shares. + +* Services that use this: + + ``nova-compute`` + +* Related options: + + vzstorage_mount_opts may include more detailed cache options. +""" + ), + cfg.ListOpt('vzstorage_mount_opts', + default=[], + help=""" +Extra mount options for pstorage-mount + +For full description of them, see +https://static.openvz.org/vz-man/man1/pstorage-mount.1.gz.html +Format is a python string representation of arguments list, like: +"[\'-v\', \'-R\', \'500\']" +Shouldn\'t include -c, -l, -C, -u, -g and -m as those have +explicit vzstorage_* options. + +* Services that use this: + + ``nova-compute`` + +* Related options: + + All other vzstorage_* options +""" + ), +] + ALL_OPTS = list(itertools.chain( libvirt_general_opts, libvirt_imagebackend_opts, @@ -430,7 +567,8 @@ ALL_OPTS = list(itertools.chain( libvirt_volume_quobyte_opts, libvirt_volume_scality_opts, libvirt_volume_smbfs_opts, - libvirt_remotefs_opts + libvirt_remotefs_opts, + libvirt_volume_vzstorage_opts, )) diff --git a/nova/tests/unit/virt/libvirt/volume/test_vzstorage.py b/nova/tests/unit/virt/libvirt/volume/test_vzstorage.py new file mode 100644 index 000000000000..f6faacd6ffd1 --- /dev/null +++ b/nova/tests/unit/virt/libvirt/volume/test_vzstorage.py @@ -0,0 +1,109 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import os + +from nova import exception +from nova.tests.unit.virt.libvirt.volume import test_volume +from nova import utils +from nova.virt.libvirt.volume import vzstorage +from os_brick.initiator import connector + + +class LibvirtVZStorageTestCase(test_volume.LibvirtVolumeBaseTestCase): + """Tests the libvirt vzstorage volume driver.""" + + def setUp(self): + super(LibvirtVZStorageTestCase, self).setUp() + self.mnt_base = '/mnt' + self.flags(vzstorage_mount_point_base=self.mnt_base, group='libvirt') + self.flags(vzstorage_cache_path="/tmp/ssd-cache/%(cluster_name)s", + group='libvirt') + + def test_libvirt_vzstorage_driver(self): + libvirt_driver = vzstorage.LibvirtVZStorageVolumeDriver(self.fake_conn) + self.assertIsInstance(libvirt_driver.connector, + connector.RemoteFsConnector) + + def test_libvirt_vzstorage_driver_opts_negative(self): + """Test that custom options cannot duplicate the configured""" + bad_opts = [ + ["-c", "clus111", "-v"], + ["-l", "/var/log/pstorage.log", "-L", "5x5"], + ["-u", "user1", "-p", "pass1"], + ["-v", "-R", "100", "-C", "/ssd"], + ] + for opts in bad_opts: + self.flags(vzstorage_mount_opts=opts, group='libvirt') + self.assertRaises(exception.NovaException, + vzstorage.LibvirtVZStorageVolumeDriver, + self.fake_conn) + + def test_libvirt_vzstorage_driver_share_fmt_neg(self): + + drv = vzstorage.LibvirtVZStorageVolumeDriver(self.fake_conn) + + wrong_export_string = 'mds1, mds2:/testcluster:passwd12111' + connection_info = {'data': {'export': wrong_export_string, + 'name': self.name}} + + err_pattern = ("^Valid share format is " + "\[mds\[,mds1\[\.\.\.\]\]:/\]clustername\[:password\]$") + self.assertRaisesRegex(exception.InvalidVolume, + err_pattern, + drv.connect_volume, + connection_info, + self.disk_info) + + def test_libvirt_vzstorage_driver_connect(self): + def brick_conn_vol(data): + return {'path': 'vstorage://testcluster'} + + drv = vzstorage.LibvirtVZStorageVolumeDriver(self.fake_conn) + drv.connector.connect_volume = brick_conn_vol + + export_string = 'testcluster' + connection_info = {'data': {'export': export_string, + 'name': self.name}} + + drv.connect_volume(connection_info, self.disk_info) + self.assertEqual('vstorage://testcluster', + connection_info['data']['device_path']) + self.assertEqual('-u stack -g qemu -m 0770 ' + '-l /var/log/pstorage/testcluster/nova.log.gz ' + '-C /tmp/ssd-cache/testcluster', + connection_info['data']['options']) + + def test_libvirt_vzstorage_driver_disconnect(self): + drv = vzstorage.LibvirtVZStorageVolumeDriver(self.fake_conn) + drv.connector.disconnect_volume = mock.MagicMock() + conn = {'data': self.disk_info} + drv.disconnect_volume(conn, self.disk_info) + drv.connector.disconnect_volume.assert_called_once_with( + self.disk_info, None) + + def test_libvirt_vzstorage_driver_get_config(self): + libvirt_driver = vzstorage.LibvirtVZStorageVolumeDriver(self.fake_conn) + export_string = 'vzstorage' + export_mnt_base = os.path.join(self.mnt_base, + utils.get_hash_str(export_string)) + file_path = os.path.join(export_mnt_base, self.name) + + connection_info = {'data': {'export': export_string, + 'name': self.name, + 'device_path': file_path}} + conf = libvirt_driver.get_config(connection_info, self.disk_info) + self.assertEqual('file', conf.source_type) + self.assertEqual(file_path, conf.source_path) + self.assertEqual('raw', conf.driver_format) + self.assertEqual('writethrough', conf.driver_cache) diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index e665ba15a155..a6361194ff87 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -165,6 +165,8 @@ libvirt_volume_drivers = [ 'hgst=nova.virt.libvirt.volume.hgst.LibvirtHGSTVolumeDriver', 'scaleio=nova.virt.libvirt.volume.scaleio.LibvirtScaleIOVolumeDriver', 'disco=nova.virt.libvirt.volume.disco.LibvirtDISCOVolumeDriver', + 'vzstorage=' + 'nova.virt.libvirt.volume.vzstorage.LibvirtVZStorageVolumeDriver', ] diff --git a/nova/virt/libvirt/volume/vzstorage.py b/nova/virt/libvirt/volume/vzstorage.py new file mode 100644 index 000000000000..e073338f4182 --- /dev/null +++ b/nova/virt/libvirt/volume/vzstorage.py @@ -0,0 +1,127 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import re + +from os_brick.initiator import connector +from oslo_config import cfg +from oslo_log import log as logging + +from nova import exception +from nova.i18n import _ +from nova import utils +from nova.virt.libvirt.volume import fs + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + +VzShare = collections.namedtuple('VzShare', + ['cluster_name', 'mds_list', 'password']) + + +class LibvirtVZStorageVolumeDriver(fs.LibvirtBaseFileSystemVolumeDriver): + """Class implements libvirt part of volume driver for VzStorage.""" + + SHARE_FORMAT_REGEX = r'(?:(\S+):/)?([a-zA-Z0-9_-]+)(?::(\S+))?$' + + def __init__(self, connection): + super(LibvirtVZStorageVolumeDriver, self).__init__(connection) + + # Check for duplicate options: + # -c - cluster name + # -l - log file, includes %(cluster_name)s, so it's handled as a + # separate config parameter + # -C - SSD cache file, the same thing with %(cluster_name)s + # -u, -g, -m - there are default values for these options, so + # they're separate config parameters + cfg_opts_set = set(CONF.libvirt.vzstorage_mount_opts) + invalid_opts_set = set(('-c', '-l', '-C', '-u', '-g', '-m',)) + invalid_cfg_opts = cfg_opts_set.intersection(invalid_opts_set) + + if invalid_cfg_opts: + msg = (_("You can't use %s options in vzstorage_mount_opts " + "configuration parameter.") % + ', '.join(invalid_cfg_opts)) + raise exception.NovaException(msg) + + # Call the factory here so we can support + # more than x86 architectures. + self.connector = connector.InitiatorConnector.factory( + 'vzstorage', utils.get_root_helper(), + vzstorage_mount_point_base=CONF.libvirt.vzstorage_mount_point_base) + + def _get_mount_point_base(self): + return CONF.libvirt.vzstorage_mount_point_base + + def get_config(self, connection_info, disk_info): + """Returns xml for libvirt.""" + conf = super(LibvirtVZStorageVolumeDriver, + self).get_config(connection_info, disk_info) + + conf.source_type = 'file' + conf.driver_cache = 'writethrough' + conf.source_path = connection_info['data']['device_path'] + conf.driver_format = connection_info['data'].get('format', 'raw') + return conf + + def _parse_vz_share(self, vz_share): + m = re.match(self.SHARE_FORMAT_REGEX, vz_share) + if not m: + msg = _("Valid share format is " + "[mds[,mds1[...]]:/]clustername[:password]") + raise exception.InvalidVolume(msg) + + if m.group(1): + mds_list = m.group(1).split(',') + else: + mds_list = None + + return VzShare(cluster_name=m.group(2), + mds_list=mds_list, + password=m.group(3)) + + def _get_mount_opts(self, vz_share): + cluster_name = self._parse_vz_share(vz_share).cluster_name + + # pstorage-mount man page: + # https://static.openvz.org/vz-man/man1/pstorage-mount.1.gz.html + mount_opts = ['-u', CONF.libvirt.vzstorage_mount_user, + '-g', CONF.libvirt.vzstorage_mount_group, + '-m', CONF.libvirt.vzstorage_mount_perms, + '-l', (CONF.libvirt.vzstorage_log_path % + {'cluster_name': cluster_name})] + + if CONF.libvirt.vzstorage_cache_path: + mount_opts.extend(['-C', (CONF.libvirt.vzstorage_cache_path % + {'cluster_name': cluster_name})]) + mount_opts.extend(CONF.libvirt.vzstorage_mount_opts) + + return ' '.join(mount_opts) + + def connect_volume(self, connection_info, disk_info): + """Attach the volume to instance_name.""" + + LOG.debug("Calling os-brick to mount vzstorage") + vz_share = connection_info['data']['export'] + connection_info['data']['options'] = self._get_mount_opts(vz_share) + device_info = self.connector.connect_volume(connection_info['data']) + LOG.debug("Attached vzstorage volume %s", device_info) + + connection_info['data']['device_path'] = device_info['path'] + + def disconnect_volume(self, connection_info, disk_dev): + """Detach the volume from instance_name.""" + + LOG.debug("calling os-brick to detach Vzstorage Volume") + self.connector.disconnect_volume(connection_info['data'], None) + LOG.debug("Disconnected Vzstorage Volume %s", disk_dev)