1a8ea0eac4
When rebuilding a volume backed instance, while copying the new
image to the existing volume, we preserve sparseness.
This could be problematic since we don't write the zero blocks of
the new image and the data in the old image can still persist
leading to a data leak scenario.
To prevent this, we are using `-S 0`[1][2] option with the `qemu-img convert`
command to write all the zero bytes into the volume.
In the testing done, this doesn't seem to be a problem with known 'raw'
images but good to handle the case anyway.
Following is the testing performed with 3 images:
1. CIRROS QCOW2 to RAW
======================
Volume size: 1 GiB
Image size (raw): 112 MiB
CREATE VOLUME FROM IMAGE (without -S 0)
LVS (10.94% allocated)
volume-91ea43ef-684c-402f-896e-63e45e5f4fff stack-volumes-lvmdriver-1 Vwi-a-tz-- 1.00g stack-volumes-lvmdriver-1-pool 10.94
REBUILD (with -S 0)
LVS (10.94% allocated)
volume-91ea43ef-684c-402f-896e-63e45e5f4fff stack-volumes-lvmdriver-1 Vwi-aotz-- 1.00g stack-volumes-lvmdriver-1-pool 10.94
Conclusion:
Same space is consumed on the disk with and without preserving sparseness.
2. DEBIAN QCOW2 to RAW
======================
Volume size: 3 GiB
Image size (raw): 2 GiB
CREATE VOLUME FROM IMAGE (without -S 0)
LVS (66.67% allocated)
volume-edc42b6a-df5d-420e-85d3-b3e52bcb735e stack-volumes-lvmdriver-1 Vwi-a-tz-- 3.00g stack-volumes-lvmdriver-1-pool 66.67
REBUILD (with -S 0)
LVS (66.67% allocated)
volume-edc42b6a-df5d-420e-85d3-b3e52bcb735e stack-volumes-lvmdriver-1 Vwi-aotz-- 3.00g stack-volumes-lvmdriver-1-pool 66.67
Conclusion:
Same space is consumed on the disk with and without preserving sparseness.
3. FEDORA QCOW2 TO RAW
======================
CREATE VOLUME FROM IMAGE (without -S 0)
Volume size: 6 GiB
Image size (raw): 5 GiB
LVS (83.33% allocated)
volume-efa1a227-a30d-4385-867a-db22a3e80ad7 stack-volumes-lvmdriver-1 Vwi-a-tz-- 6.00g stack-volumes-lvmdriver-1-pool 83.33
REBUILD (with -S 0)
LVS (83.33% allocated)
volume-efa1a227-a30d-4385-867a-db22a3e80ad7 stack-volumes-lvmdriver-1 Vwi-aotz-- 6.00g stack-volumes-lvmdriver-1-pool 83.33
Conclusion:
Same space is consumed on the disk with and without preserving sparseness.
Another testing was done to check if the `-S 0` option actually
works in OpenStack setup.
Note that we are converting qcow2 to qcow2 image which won't
happen in a real world deployment and only for test purposes.
DEBIAN QCOW2 TO QCOW2
=====================
CREATE VOLUME FROM IMAGE (without -S 0)
LVS (52.61% allocated)
volume-de581f84-e722-4f4a-94fb-10f767069f50 stack-volumes-lvmdriver-1 Vwi-a-tz-- 3.00g stack-volumes-lvmdriver-1-pool 52.61
REBUILD (with -S 0)
LVS (66.68% allocated)
volume-de581f84-e722-4f4a-94fb-10f767069f50 stack-volumes-lvmdriver-1 Vwi-aotz-- 3.00g stack-volumes-lvmdriver-1-pool 66.68
Conclusion:
We can see that the space allocation increased hence we are not preserving sparseness when using the -S 0 option.
[1] https://qemu-project.gitlab.io/qemu/tools/qemu-img.html#cmdoption-qemu-img-common-opts-S
[2] abf635ddfe/qemu-img.c (L182-L186)
Closes-Bug: #2045431
Change-Id: I5be7eaba68a5b8e1c43f0d95486b5c79c14e1b95
415 lines
15 KiB
Python
415 lines
15 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import json
|
|
|
|
from os_brick import initiator
|
|
from os_brick.initiator import connector
|
|
from oslo_log import log as logging
|
|
from oslo_utils import importutils
|
|
from oslo_utils import units
|
|
import requests
|
|
|
|
from cinder.common import constants
|
|
from cinder import context
|
|
from cinder import exception
|
|
from cinder.i18n import _
|
|
from cinder.image import image_utils
|
|
from cinder import interface
|
|
from cinder import utils
|
|
from cinder.volume import driver
|
|
from cinder.volume import volume_utils
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
@interface.volumedriver
|
|
class SPDKDriver(driver.VolumeDriver):
|
|
"""Executes commands relating to Volumes."""
|
|
|
|
VERSION = '1.0.0'
|
|
|
|
# ThirdPartySystems wiki page
|
|
CI_WIKI_NAME = "Mellanox_CI"
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
# Parent sets db, host, _execute and base config
|
|
super(SPDKDriver, self).__init__(*args, **kwargs)
|
|
|
|
self.lvs = []
|
|
self.ctxt = context.get_admin_context()
|
|
|
|
target_driver = (
|
|
self.target_mapping[self.configuration.safe_get('target_helper')])
|
|
|
|
LOG.debug('SPDK attempting to initialize LVM driver with the '
|
|
'following target_driver: %s',
|
|
target_driver)
|
|
|
|
self.target_driver = importutils.import_object(
|
|
target_driver,
|
|
configuration=self.configuration,
|
|
executor=self._execute)
|
|
|
|
@staticmethod
|
|
def get_driver_options():
|
|
return []
|
|
|
|
def _rpc_call(self, method, params=None):
|
|
payload = {}
|
|
payload['jsonrpc'] = '2.0'
|
|
payload['id'] = 1
|
|
payload['method'] = method
|
|
if params is not None:
|
|
payload['params'] = params
|
|
|
|
req = requests.post(self.url,
|
|
data=json.dumps(payload),
|
|
auth=(self.configuration.spdk_rpc_username,
|
|
self.configuration.spdk_rpc_password),
|
|
verify=self.configuration.driver_ssl_cert_verify,
|
|
timeout=30)
|
|
|
|
if not req.ok:
|
|
raise exception.VolumeBackendAPIException(
|
|
data=_('SPDK target responded with error: %s') % req.text)
|
|
|
|
return req.json()['result']
|
|
|
|
def _update_volume_stats(self):
|
|
"""Retrieve stats info from volume group."""
|
|
|
|
LOG.debug('SPDK Updating volume stats')
|
|
status = {'volume_backend_name': 'SPDK',
|
|
'vendor_name': 'Open Source',
|
|
'driver_version': self.VERSION,
|
|
'storage_protocol': constants.NVMEOF}
|
|
pools_status = []
|
|
self.lvs = []
|
|
|
|
output = self._rpc_call('bdev_lvol_get_lvstores')
|
|
if output:
|
|
for lvs in output:
|
|
pool = {}
|
|
lvs_entry = {}
|
|
free_size = (lvs['free_clusters']
|
|
* lvs['cluster_size']
|
|
/ units.Gi)
|
|
total_size = (lvs['total_data_clusters']
|
|
* lvs['cluster_size']
|
|
/ units.Gi)
|
|
pool["volume_backend_name"] = 'SPDK'
|
|
pool["vendor_name"] = 'Open Source'
|
|
pool["driver_version"] = self.VERSION
|
|
pool["storage_protocol"] = constants.NVMEOF
|
|
pool["total_capacity_gb"] = total_size
|
|
pool["free_capacity_gb"] = free_size
|
|
pool["pool_name"] = lvs['name']
|
|
pools_status.append(pool)
|
|
|
|
lvs_entry['name'] = lvs['name']
|
|
lvs_entry['uuid'] = lvs['uuid']
|
|
lvs_entry['free_size'] = free_size
|
|
lvs_entry['total_size'] = total_size
|
|
self.lvs.append(lvs_entry)
|
|
|
|
status['pools'] = pools_status
|
|
self._stats = status
|
|
|
|
for lvs in self.lvs:
|
|
LOG.debug('SPDK lvs name: %s, total space: %s, free space: %s',
|
|
lvs['name'],
|
|
lvs['total_size'],
|
|
lvs['free_size'])
|
|
|
|
def _get_spdk_volume_name(self, name):
|
|
output = self._rpc_call('bdev_get_bdevs')
|
|
for bdev in output:
|
|
for alias in bdev['aliases']:
|
|
if name in alias:
|
|
return bdev['name']
|
|
|
|
def _get_spdk_lvs_uuid(self, spdk_name):
|
|
output = self._rpc_call('bdev_get_bdevs')
|
|
for bdev in output:
|
|
if spdk_name in bdev['name']:
|
|
return bdev['driver_specific']['lvol']['lvol_store_uuid']
|
|
|
|
def _get_spdk_lvs_free_space(self, lvs_uuid):
|
|
self._update_volume_stats()
|
|
|
|
for lvs in self.lvs:
|
|
if lvs_uuid in lvs['uuid']:
|
|
return lvs['free_size']
|
|
|
|
return 0
|
|
|
|
def _delete_bdev(self, name):
|
|
spdk_name = self._get_spdk_volume_name(name)
|
|
if spdk_name is not None:
|
|
params = {'name': spdk_name}
|
|
self._rpc_call('bdev_lvol_delete', params)
|
|
LOG.debug('SPDK bdev %s deleted', spdk_name)
|
|
else:
|
|
LOG.debug('Could not find volume %s using SPDK driver', name)
|
|
|
|
def _create_volume(self, volume, snapshot=None):
|
|
output = self._rpc_call('bdev_lvol_get_lvstores')
|
|
for lvs in output:
|
|
free_size = (lvs['free_clusters'] * lvs['cluster_size'])
|
|
if free_size / units.Gi >= volume.size:
|
|
if snapshot is None:
|
|
params = {
|
|
'lvol_name': volume.name,
|
|
'size': volume.size * units.Gi,
|
|
'uuid': lvs['uuid']}
|
|
output2 = self._rpc_call('bdev_lvol_create', params)
|
|
else:
|
|
snapshot_spdk_name = (
|
|
self._get_spdk_volume_name(snapshot.name))
|
|
params = {
|
|
'clone_name': volume.name,
|
|
'snapshot_name': snapshot_spdk_name}
|
|
output2 = self._rpc_call('bdev_lvol_clone', params)
|
|
spdk_name = self._get_spdk_volume_name(volume.name)
|
|
params = {'name': spdk_name}
|
|
self._rpc_call('bdev_lvol_inflate', params)
|
|
|
|
if volume.size > snapshot.volume_size:
|
|
params = {'name': spdk_name,
|
|
'size': volume.size * units.Gi}
|
|
self._rpc_call('bdev_lvol_resize', params)
|
|
|
|
LOG.debug('SPDK created lvol: %s', output2)
|
|
|
|
return
|
|
|
|
LOG.error('Unable to create volume using SPDK - no resources found')
|
|
raise exception.VolumeBackendAPIException(
|
|
data=_('Unable to create volume using SPDK'
|
|
' - no resources found'))
|
|
|
|
def do_setup(self, context):
|
|
try:
|
|
payload = {'method': 'bdev_get_bdevs', 'jsonrpc': '2.0', 'id': 1}
|
|
self.url = ('%(protocol)s://%(ip)s:%(port)s/' %
|
|
{'protocol': self.configuration.spdk_rpc_protocol,
|
|
'ip': self.configuration.spdk_rpc_ip,
|
|
'port': self.configuration.spdk_rpc_port})
|
|
requests.post(self.url,
|
|
data=json.dumps(payload),
|
|
auth=(self.configuration.spdk_rpc_username,
|
|
self.configuration.spdk_rpc_password),
|
|
verify=self.configuration.driver_ssl_cert_verify,
|
|
timeout=30)
|
|
except Exception as err:
|
|
err_msg = (
|
|
_('Could not connect to SPDK target: %(err)s')
|
|
% {'err': err})
|
|
LOG.error(err_msg)
|
|
raise exception.VolumeBackendAPIException(data=err_msg)
|
|
|
|
def check_for_setup_error(self):
|
|
"""Verify that requirements are in place to use LVM driver."""
|
|
|
|
# If configuration is incorrect we will get exception here
|
|
self._rpc_call('bdev_get_bdevs')
|
|
|
|
def create_volume(self, volume):
|
|
"""Creates a logical volume."""
|
|
LOG.debug('SPDK create volume')
|
|
|
|
return self._create_volume(volume)
|
|
|
|
def delete_volume(self, volume):
|
|
"""Deletes a logical volume."""
|
|
LOG.debug('SPDK deleting volume %s', volume.name)
|
|
|
|
self._delete_bdev(volume.name)
|
|
|
|
def create_volume_from_snapshot(self, volume, snapshot):
|
|
"""Creates a volume from a snapshot."""
|
|
|
|
free_size = self._get_spdk_lvs_free_space(
|
|
self._get_spdk_lvs_uuid(
|
|
self._get_spdk_volume_name(snapshot.name)))
|
|
|
|
if free_size < volume.size:
|
|
raise exception.VolumeBackendAPIException(
|
|
data=_('Not enough space to create snapshot with SPDK'))
|
|
|
|
return self._create_volume(volume, snapshot)
|
|
|
|
def create_snapshot(self, snapshot):
|
|
"""Creates a snapshot."""
|
|
volume = snapshot['volume']
|
|
spdk_name = self._get_spdk_volume_name(volume.name)
|
|
|
|
if spdk_name is None:
|
|
raise exception.VolumeBackendAPIException(
|
|
data=_('Could not create snapshot with SPDK driver'))
|
|
|
|
free_size = self._get_spdk_lvs_free_space(
|
|
self._get_spdk_lvs_uuid(spdk_name))
|
|
|
|
if free_size < volume.size:
|
|
raise exception.VolumeBackendAPIException(
|
|
data=_('Not enough space to create snapshot with SPDK'))
|
|
|
|
params = {
|
|
'lvol_name': spdk_name,
|
|
'snapshot_name': snapshot['name']}
|
|
self._rpc_call('bdev_lvol_snapshot', params)
|
|
|
|
params = {'name': spdk_name}
|
|
self._rpc_call('bdev_lvol_inflate', params)
|
|
|
|
def delete_snapshot(self, snapshot):
|
|
"""Deletes a snapshot."""
|
|
spdk_name = self._get_spdk_volume_name(snapshot.name)
|
|
if spdk_name is None:
|
|
return
|
|
|
|
params = {'name': spdk_name}
|
|
bdev = self._rpc_call('bdev_get_bdevs', params)
|
|
if 'clones' in bdev[0]['driver_specific']['lvol']:
|
|
for clone in bdev[0]['driver_specific']['lvol']['clones']:
|
|
spdk_name = self._get_spdk_volume_name(clone)
|
|
params = {'name': spdk_name}
|
|
self._rpc_call('bdev_lvol_inflate', params)
|
|
|
|
self._delete_bdev(snapshot.name)
|
|
|
|
def create_cloned_volume(self, volume, src_volume):
|
|
spdk_name = self._get_spdk_volume_name(src_volume.name)
|
|
|
|
free_size = self._get_spdk_lvs_free_space(
|
|
self._get_spdk_lvs_uuid(spdk_name))
|
|
|
|
# We need additional space for snapshot that will be used here
|
|
if free_size < 2 * src_volume.size + volume.size:
|
|
raise exception.VolumeBackendAPIException(
|
|
data=_('Not enough space to clone volume with SPDK'))
|
|
|
|
snapshot_name = 'snp-' + src_volume.name
|
|
|
|
params = {
|
|
'lvol_name': spdk_name,
|
|
'snapshot_name': snapshot_name}
|
|
self._rpc_call('bdev_lvol_snapshot', params)
|
|
|
|
params = {'name': spdk_name}
|
|
self._rpc_call('bdev_lvol_inflate', params)
|
|
|
|
snapshot_spdk_name = self._get_spdk_volume_name(snapshot_name)
|
|
params = {
|
|
'clone_name': volume.name,
|
|
'snapshot_name': snapshot_spdk_name}
|
|
|
|
self._rpc_call('bdev_lvol_clone', params)
|
|
|
|
spdk_name = self._get_spdk_volume_name(volume.name)
|
|
params = {'name': spdk_name}
|
|
self._rpc_call('bdev_lvol_inflate', params)
|
|
|
|
self._delete_bdev(snapshot_name)
|
|
|
|
if volume.size > src_volume.size:
|
|
self.extend_volume(volume, volume.size)
|
|
|
|
def copy_image_to_volume(self, context, volume, image_service, image_id,
|
|
disable_sparse=False):
|
|
"""Fetch the image from image_service and write it to the volume."""
|
|
|
|
volume['provider_location'] = (
|
|
self.create_export(context, volume, None)['provider_location'])
|
|
connection_data = self.initialize_connection(volume, None)['data']
|
|
target_connector = (
|
|
connector.InitiatorConnector.factory(initiator.NVME,
|
|
utils.get_root_helper()))
|
|
|
|
try:
|
|
device_info = target_connector.connect_volume(connection_data)
|
|
except Exception:
|
|
LOG.info('Could not connect SPDK target device')
|
|
return
|
|
|
|
connection_data['device_path'] = device_info['path']
|
|
|
|
try:
|
|
image_utils.fetch_to_raw(context,
|
|
image_service,
|
|
image_id,
|
|
device_info['path'],
|
|
self.configuration.volume_dd_blocksize,
|
|
size=volume['size'],
|
|
disable_sparse=disable_sparse)
|
|
|
|
finally:
|
|
target_connector.disconnect_volume(connection_data, volume)
|
|
|
|
def copy_volume_to_image(self, context, volume, image_service, image_meta):
|
|
"""Copy the volume to the specified image."""
|
|
volume['provider_location'] = (
|
|
self.create_export(context, volume, None)['provider_location'])
|
|
connection_data = self.initialize_connection(volume, None)['data']
|
|
target_connector = (
|
|
connector.InitiatorConnector.factory(initiator.NVME,
|
|
utils.get_root_helper()))
|
|
|
|
try:
|
|
device_info = target_connector.connect_volume(connection_data)
|
|
except Exception:
|
|
LOG.info('Could not connect SPDK target device')
|
|
return
|
|
|
|
connection_data['device_path'] = device_info['path']
|
|
|
|
try:
|
|
volume_utils.upload_volume(context,
|
|
image_service,
|
|
image_meta,
|
|
device_info['path'],
|
|
volume)
|
|
finally:
|
|
target_connector.disconnect_volume(connection_data, volume)
|
|
|
|
def extend_volume(self, volume, new_size):
|
|
"""Extend an existing volume's size."""
|
|
spdk_name = self._get_spdk_volume_name(volume.name)
|
|
params = {'name': spdk_name, 'size': new_size * units.Gi}
|
|
self._rpc_call('bdev_lvol_resize', params)
|
|
|
|
# ####### Interface methods for DataPath (Target Driver) ########
|
|
def ensure_export(self, context, volume):
|
|
pass
|
|
|
|
def create_export(self, context, volume, connector, vg=None):
|
|
export_info = self.target_driver.create_export(
|
|
context,
|
|
volume,
|
|
None)
|
|
return {'provider_location': export_info['location'],
|
|
'provider_auth': export_info['auth'], }
|
|
|
|
def remove_export(self, context, volume):
|
|
self.target_driver.remove_export(context, volume)
|
|
|
|
def initialize_connection(self, volume, connector, **kwargs):
|
|
return self.target_driver.initialize_connection(volume, connector)
|
|
|
|
def validate_connector(self, connector):
|
|
return self.target_driver.validate_connector(connector)
|
|
|
|
def terminate_connection(self, volume, connector, **kwargs):
|
|
pass
|