openstack
/
cinder
Code Issues Proposed changes

Merge "Set volume_dd_blocksize configurable on per-driver basis"

This commit is contained in:
Jenkins 2013-12-23 18:46:36 +00:00 committed by Gerrit Code Review
parent 3364034235 5654992881
commit 8a5ce963dc
22 changed files with 107 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cinder/image/image_utils.py
View File

@ -114,21 +114,21 @@ def fetch_verify_image(context, image_service, image_id, dest,
def fetch_to_vhd(context, image_service,
image_id, dest,
image_id, dest, blocksize,
user_id=None, project_id=None):
fetch_to_volume_format(context, image_service, image_id, dest, 'vpc',
user_id, project_id)
blocksize, user_id, project_id)
def fetch_to_raw(context, image_service,
image_id, dest,
image_id, dest, blocksize,
user_id=None, project_id=None, size=None):
fetch_to_volume_format(context, image_service, image_id, dest, 'raw',
user_id, project_id, size)
blocksize, user_id, project_id, size)
def fetch_to_volume_format(context, image_service,
image_id, dest, volume_format,
image_id, dest, volume_format, blocksize,
user_id=None, project_id=None, size=None):
if (CONF.image_conversion_dir and not
os.path.exists(CONF.image_conversion_dir)):
@ -180,7 +180,7 @@ def fetch_to_volume_format(context, image_service,
LOG.debug(_('Copying image from %(tmp)s to volume %(dest)s - '
'size: %(size)s') % {'tmp': tmp, 'dest': dest,
'size': image_meta['size']})
volume_utils.copy_volume(tmp, dest, image_meta['size'])
volume_utils.copy_volume(tmp, dest, image_meta['size'], blocksize)
return
data = qemu_img_info(tmp)

5
cinder/tests/test_block_device.py
View File

@ -137,7 +137,7 @@ class TestBlockDeviceDriver(cinder.test.TestCase):
self.mox.StubOutWithMock(self.drv, '_get_device_size')
self.drv.local_path(TEST_SRC).AndReturn('/dev/loop1')
self.drv._get_device_size('/dev/loop2').AndReturn(1)
volutils.copy_volume('/dev/loop1', dev, 2048,
volutils.copy_volume('/dev/loop1', dev, 2048, mox.IgnoreArg(),
execute=self.drv._execute)
self.mox.ReplayAll()
self.assertEqual(self.drv.create_cloned_volume(TEST_VOLUME, TEST_SRC),
@ -152,7 +152,8 @@ class TestBlockDeviceDriver(cinder.test.TestCase):
self.mox.StubOutWithMock(self.drv, 'local_path')
self.drv.local_path(TEST_VOLUME).AndReturn('/dev/loop1')
image_utils.fetch_to_raw(context, TEST_IMAGE_SERVICE,
TEST_IMAGE_ID, '/dev/loop1', size=1)
TEST_IMAGE_ID, '/dev/loop1', mox.IgnoreArg(),
size=1)
self.mox.ReplayAll()
self.drv.copy_image_to_volume(context, TEST_VOLUME, TEST_IMAGE_SERVICE,
TEST_IMAGE_ID)

2
cinder/tests/test_coraid.py
View File

@ -245,6 +245,7 @@ class CoraidDriverTestCase(test.TestCase):
configuration.coraid_repository_key = fake_coraid_repository_key
configuration.use_multipath_for_image_xfer = False
configuration.num_volume_device_scan_tries = 3
configuration.volume_dd_blocksize = '1M'
self.fake_rpc = FakeRpc()
self.stubs.Set(coraid.CoraidRESTClient, 'rpc', self.fake_rpc)
@ -829,6 +830,7 @@ class CoraidDriverImageTestCases(CoraidDriverTestCase):
fake_image_service,
fake_image_id,
self.fake_dev_path,
mox.IgnoreArg(),
size=fake_volume_size)
self.mox.ReplayAll()

17
cinder/tests/test_image_utils.py
View File

@ -251,7 +251,8 @@ class TestUtils(test.TestCase):
self._test_fetch_to_raw(src_inf=SRC_INFO, dest_inf=DST_INFO)
image_utils.fetch_to_raw(context, self._image_service,
self.TEST_IMAGE_ID, self.TEST_DEV_PATH)
self.TEST_IMAGE_ID, self.TEST_DEV_PATH,
mox.IgnoreArg())
self._mox.VerifyAll()
def test_fetch_to_raw_no_qemu_img(self):
@ -260,7 +261,8 @@ class TestUtils(test.TestCase):
self.assertRaises(exception.ImageUnacceptable,
image_utils.fetch_to_raw,
context, self._image_service,
self.TEST_IMAGE_ID, self.TEST_DEV_PATH)
self.TEST_IMAGE_ID, self.TEST_DEV_PATH,
mox.IgnoreArg())
self._mox.VerifyAll()
@ -275,7 +277,8 @@ class TestUtils(test.TestCase):
self.assertRaises(exception.ImageUnacceptable,
image_utils.fetch_to_raw,
context, self._image_service,
self.TEST_IMAGE_ID, self.TEST_DEV_PATH)
self.TEST_IMAGE_ID, self.TEST_DEV_PATH,
mox.IgnoreArg())
self._mox.VerifyAll()
def test_fetch_to_raw_on_error_backing_file(self):
@ -291,7 +294,8 @@ class TestUtils(test.TestCase):
self.assertRaises(exception.ImageUnacceptable,
image_utils.fetch_to_raw,
context, self._image_service,
self.TEST_IMAGE_ID, self.TEST_DEV_PATH)
self.TEST_IMAGE_ID, self.TEST_DEV_PATH,
mox.IgnoreArg())
self._mox.VerifyAll()
def test_fetch_to_raw_on_error_not_convert_to_raw(self):
@ -306,7 +310,8 @@ class TestUtils(test.TestCase):
self.assertRaises(exception.ImageUnacceptable,
image_utils.fetch_to_raw,
context, self._image_service,
self.TEST_IMAGE_ID, self.TEST_DEV_PATH)
self.TEST_IMAGE_ID, self.TEST_DEV_PATH,
mox.IgnoreArg())
def test_fetch_to_raw_on_error_image_size(self):
TEST_VOLUME_SIZE = 1
@ -322,7 +327,7 @@ class TestUtils(test.TestCase):
image_utils.fetch_to_raw,
context, self._image_service,
self.TEST_IMAGE_ID, self.TEST_DEV_PATH,
size=TEST_VOLUME_SIZE)
mox.IgnoreArg(), size=TEST_VOLUME_SIZE)
def _test_fetch_verify_image(self, qemu_info, volume_size=1):
fake_image_service = FakeImageService()

2
cinder/tests/test_nfs.py
View File

@ -135,6 +135,7 @@ class NfsDriverTestCase(test.TestCase):
self.configuration.nfs_oversub_ratio = 1.0
self.configuration.nfs_mount_point_base = self.TEST_MNT_POINT_BASE
self.configuration.nfs_mount_options = None
self.configuration.volume_dd_blocksize = '1M'
self._driver = nfs.NfsDriver(configuration=self.configuration)
self._driver.shares = {}
self.addCleanup(self.stubs.UnsetAll)
@ -174,6 +175,7 @@ class NfsDriverTestCase(test.TestCase):
mox.StubOutWithMock(image_utils, 'fetch_to_raw')
image_utils.fetch_to_raw(None, None, None, TEST_IMG_SOURCE,
mox_lib.IgnoreArg(),
size=self.TEST_SIZE_IN_GB)
mox.StubOutWithMock(image_utils, 'resize_image')

4
cinder/tests/test_rbd.py
View File

@ -97,6 +97,7 @@ class RBDTestCase(test.TestCase):
self.configuration.rbd_secret_uuid = None
self.configuration.rbd_user = None
self.configuration.append_config_values(mox.IgnoreArg())
self.configuration.volume_dd_blocksize = '1M'
self.rados = self.mox.CreateMockAnything()
self.rbd = self.mox.CreateMockAnything()
@ -370,7 +371,8 @@ class RBDTestCase(test.TestCase):
self.name = name
yield FakeTmp('test')
def fake_fetch_to_raw(ctx, image_service, image_id, path, size=None):
def fake_fetch_to_raw(ctx, image_service, image_id, path, blocksize,
size=None):
pass
self.stubs.Set(tempfile, 'NamedTemporaryFile', fake_temp_file)

2
cinder/tests/test_scality.py
View File

@ -96,6 +96,7 @@ class ScalityDriverTestCase(test.TestCase):
scality.CONF.scality_sofs_config = self.TEST_CONFIG
scality.CONF.scality_sofs_mount_point = self.TEST_MOUNT
scality.CONF.scality_sofs_volume_dir = self.TEST_VOLDIR
scality.CONF.volume_dd_blocksize = '1M'
def _execute_wrapper(self, cmd, *args, **kwargs):
try:
@ -238,6 +239,7 @@ class ScalityDriverTestCase(test.TestCase):
self.TEST_IMAGE_SERVICE,
self.TEST_IMAGE_ID,
self.TEST_VOLPATH,
mox_lib.IgnoreArg(),
size=self.TEST_VOLSIZE)
self.mox.ReplayAll()

14
cinder/tests/test_volume.py
View File

@ -1533,7 +1533,8 @@ class VolumeTestCase(BaseVolumeTestCase):
image_service, image_id):
pass
def fake_fetch_to_raw(ctx, image_service, image_id, path, size=None):
def fake_fetch_to_raw(ctx, image_service, image_id, path, blocksize,
size=None):
pass
def fake_clone_image(volume_ref, image_location, image_id, image_meta):
@ -2465,7 +2466,8 @@ class LVMISCSIVolumeDriverTestCase(DriverTestCase):
self.stubs.Set(self.volume.driver, '_execute', fake_execute)
self.stubs.Set(volutils, 'copy_volume',
lambda x, y, z, sync=False, execute='foo': None)
lambda x, y, z, sync=False, execute='foo',
blocksize=mox.IgnoreArg(): None)
self.stubs.Set(volutils, 'get_all_volume_groups',
get_all_volume_groups)
@ -2505,11 +2507,13 @@ class LVMVolumeDriverTestCase(DriverTestCase):
os.path.exists(mox.IgnoreArg()).AndReturn(True)
volutils.copy_volume('/dev/zero', mox.IgnoreArg(), 123 * 1024,
execute=lvm_driver._execute, sync=True)
mox.IgnoreArg(), execute=lvm_driver._execute,
sync=True)
os.path.exists(mox.IgnoreArg()).AndReturn(True)
volutils.copy_volume('/dev/zero', mox.IgnoreArg(), 123 * 1024,
execute=lvm_driver._execute, sync=True)
mox.IgnoreArg(), execute=lvm_driver._execute,
sync=True)
os.path.exists(mox.IgnoreArg()).AndReturn(True)
@ -2581,7 +2585,7 @@ class LVMVolumeDriverTestCase(DriverTestCase):
name = 'snapshot-' + uuid
mangle_name = '_' + re.sub(r'-', r'--', name)
def fake_copy_volume(srcstr, deststr, size, **kwargs):
def fake_copy_volume(srcstr, deststr, size, blocksize, **kwargs):
self.assertEqual(deststr,
'/dev/mapper/cinder--volumes-%s-cow' %
mangle_name)

18
cinder/tests/test_volume_utils.py
View File

@ -122,36 +122,30 @@ class UsageInfoTestCase(test.TestCase):
class LVMVolumeDriverTestCase(test.TestCase):
def test_convert_blocksize_option(self):
# Test valid volume_dd_blocksize
CONF.set_override('volume_dd_blocksize', '10M')
bs, count = volume_utils._calculate_count(1024)
bs, count = volume_utils._calculate_count(1024, '10M')
self.assertEqual(bs, '10M')
self.assertEqual(count, 103)
CONF.set_override('volume_dd_blocksize', '1xBBB')
bs, count = volume_utils._calculate_count(1024)
bs, count = volume_utils._calculate_count(1024, '1xBBB')
self.assertEqual(bs, '1M')
self.assertEqual(count, 1024)
# Test 'volume_dd_blocksize' with fraction
CONF.set_override('volume_dd_blocksize', '1.3M')
bs, count = volume_utils._calculate_count(1024)
bs, count = volume_utils._calculate_count(1024, '1.3M')
self.assertEqual(bs, '1M')
self.assertEqual(count, 1024)
# Test zero-size 'volume_dd_blocksize'
CONF.set_override('volume_dd_blocksize', '0M')
bs, count = volume_utils._calculate_count(1024)
bs, count = volume_utils._calculate_count(1024, '0M')
self.assertEqual(bs, '1M')
self.assertEqual(count, 1024)
# Test negative 'volume_dd_blocksize'
CONF.set_override('volume_dd_blocksize', '-1M')
bs, count = volume_utils._calculate_count(1024)
bs, count = volume_utils._calculate_count(1024, '-1M')
self.assertEqual(bs, '1M')
self.assertEqual(count, 1024)
# Test non-digital 'volume_dd_blocksize'
CONF.set_override('volume_dd_blocksize', 'ABM')
bs, count = volume_utils._calculate_count(1024)
bs, count = volume_utils._calculate_count(1024, 'ABM')
self.assertEqual(bs, '1M')
self.assertEqual(count, 1024)

3
cinder/tests/test_windows.py
View File

@ -304,7 +304,8 @@ class TestWindowsDriver(test.TestCase):
mox.StubOutWithMock(image_utils, 'fetch_to_vhd')
image_utils.fetch_to_vhd(None, None, None,
self.fake_local_path(volume))
self.fake_local_path(volume),
mox_lib.IgnoreArg())
mox.ReplayAll()

4
cinder/tests/test_xenapi_sm.py
View File

@ -47,6 +47,7 @@ def get_configured_driver(server='ignore_server', path='ignore_path'):
configuration.xenapi_nfs_server = server
configuration.xenapi_nfs_serverpath = path
configuration.append_config_values(mox.IgnoreArg())
configuration.volume_dd_blocksize = '1M'
return driver.XenAPINFSDriver(configuration=configuration)
@ -402,7 +403,8 @@ class DriverTestCase(test.TestCase):
simple_context('device'))
driver.image_utils.fetch_to_raw(
context, 'image_service', 'image_id', 'device', size=1)
context, 'image_service', 'image_id', 'device', mox.IgnoreArg(),
size=1)
mock.ReplayAll()
drv._use_image_utils_to_pipe_bytes_to_volume(

16
cinder/volume/driver.py
View File

@ -95,7 +95,12 @@ volume_opts = [
help=('Sets the behavior of the iSCSI target '
'to either perform blockio or fileio '
'optionally, auto can be set and Cinder '
'will autodetect type of backing device'))]
'will autodetect type of backing device')),
cfg.StrOpt('volume_dd_blocksize',
default='1M',
help='The default block size used when copying/clearing '
'volumes'),
]
# for backward compatibility
iser_opts = [
@ -313,9 +318,11 @@ class VolumeDriver(object):
try:
size_in_mb = int(src_vol['size']) * 1024 # vol size is in GB
volume_utils.copy_volume(src_attach_info['device']['path'],
dest_attach_info['device']['path'],
size_in_mb)
volume_utils.copy_volume(
src_attach_info['device']['path'],
dest_attach_info['device']['path'],
size_in_mb,
self.configuration.volume_dd_blocksize)
copy_error = False
except Exception:
with excutils.save_and_reraise_exception():
@ -342,6 +349,7 @@ class VolumeDriver(object):
image_service,
image_id,
attach_info['device']['path'],
self.configuration.volume_dd_blocksize,
size=volume['size'])
finally:
self._detach_volume(attach_info)

15
cinder/volume/drivers/block_device.py
View File

@ -273,8 +273,10 @@ class BlockDeviceDriver(driver.ISCSIDriver):
if self.configuration.volume_clear == 'zero':
if clear_size == 0:
return volutils.copy_volume('/dev/zero', vol_path, size_in_m,
sync=True, execute=self._execute)
return volutils.copy_volume(
'/dev/zero', vol_path, size_in_m,
self.configuration.volume_dd_blocksize,
sync=True, execute=self._execute)
else:
clear_cmd = ['shred', '-n0', '-z', '-s%dMiB' % clear_size]
elif self.configuration.volume_clear == 'shred':
@ -295,6 +297,7 @@ class BlockDeviceDriver(driver.ISCSIDriver):
image_service,
image_id,
self.local_path(volume),
self.configuration.volume_dd_blocksize,
size=volume['size'])
def copy_volume_to_image(self, context, volume, image_service, image_meta):
@ -307,9 +310,11 @@ class BlockDeviceDriver(driver.ISCSIDriver):
def create_cloned_volume(self, volume, src_vref):
LOG.info(_('Creating clone of volume: %s') % src_vref['id'])
device = self.find_appropriate_size_device(src_vref['size'])
volutils.copy_volume(self.local_path(src_vref), device,
self._get_device_size(device) * 2048,
execute=self._execute)
volutils.copy_volume(
self.local_path(src_vref), device,
self._get_device_size(device) * 2048,
self.configuration.volume_dd_blocksize,
execute=self._execute)
return {
'provider_location': self._iscsi_location(None, None, None, None,
device),

4
cinder/volume/drivers/gpfs.py
View File

@ -544,7 +544,9 @@ class GPFSDriver(driver.VolumeDriver):
LOG.debug('Copy image to vol %s using image_utils fetch_to_raw' %
volume['id'])
image_utils.fetch_to_raw(context, image_service, image_id,
self.local_path(volume), size=volume['size'])
self.local_path(volume),
self.configuration.volume_dd_blocksize,
size=volume['size'])
self._resize_volume_file(volume, volume['size'])
def _resize_volume_file(self, volume, new_size):

26
cinder/volume/drivers/lvm.py
View File

@ -173,6 +173,7 @@ class LVMVolumeDriver(driver.VolumeDriver):
volutils.copy_volume(self.local_path(snapshot),
self.local_path(volume),
snapshot['volume_size'] * 1024,
self.configuration.volume_dd_blocksize,
execute=self._execute)
def delete_volume(self, volume):
@ -229,10 +230,12 @@ class LVMVolumeDriver(driver.VolumeDriver):
if self.configuration.volume_clear == 'zero':
if size_in_m == 0:
return volutils.copy_volume('/dev/zero',
dev_path, size_in_g * 1024,
sync=True,
execute=self._execute)
return volutils.copy_volume(
'/dev/zero',
dev_path, size_in_g * 1024,
self.configuration.volume_dd_blocksize,
sync=True,
execute=self._execute)
else:
clear_cmd = ['shred', '-n0', '-z', '-s%dMiB' % size_in_m]
elif self.configuration.volume_clear == 'shred':
@ -279,7 +282,9 @@ class LVMVolumeDriver(driver.VolumeDriver):
image_utils.fetch_to_raw(context,
image_service,
image_id,
self.local_path(volume), size=volume['size'])
self.local_path(volume),
self.configuration.volume_dd_blocksize,
size=volume['size'])
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
@ -310,10 +315,12 @@ class LVMVolumeDriver(driver.VolumeDriver):
mirror_count)
try:
volutils.copy_volume(self.local_path(temp_snapshot),
self.local_path(volume),
src_vref['size'] * 1024,
execute=self._execute)
volutils.copy_volume(
self.local_path(temp_snapshot),
self.local_path(volume),
src_vref['size'] * 1024,
self.configuration.volume_dd_blocksize,
execute=self._execute)
finally:
self.delete_snapshot(temp_snapshot)
@ -725,6 +732,7 @@ class LVMISCSIDriver(LVMVolumeDriver, driver.ISCSIDriver):
volutils.copy_volume(self.local_path(volume),
self.local_path(volume, vg=dest_vg),
volume['size'],
self.configuration.volume_dd_blocksize,
execute=self._execute)
self._delete_volume(volume)
model_update = self._create_export(ctxt, volume, vg=dest_vg)

1
cinder/volume/drivers/nfs.py
View File

@ -244,6 +244,7 @@ class RemoteFsDriver(driver.VolumeDriver):
image_service,
image_id,
self.local_path(volume),
self.configuration.volume_dd_blocksize,
size=volume['size'])
# NOTE (leseb): Set the virtual size of the image

4
cinder/volume/drivers/rbd.py
View File

@ -769,7 +769,9 @@ class RBDDriver(driver.VolumeDriver):
with tempfile.NamedTemporaryFile(dir=tmp_dir) as tmp:
image_utils.fetch_to_raw(context, image_service, image_id,
tmp.name, size=volume['size'])
tmp.name,
self.configuration.volume_dd_blocksize,
size=volume['size'])
self.delete_volume(volume)

1
cinder/volume/drivers/scality.py
View File

@ -240,6 +240,7 @@ class ScalityDriver(driver.VolumeDriver):
image_service,
image_id,
self.local_path(volume),
CONF.volume_dd_blocksize,
size=volume['size'])
self.create_volume(volume)

3
cinder/volume/drivers/windows/windows.py
View File

@ -170,7 +170,8 @@ class WindowsDriver(driver.ISCSIDriver):
"""Fetch the image from image_service and write it to the volume."""
# Convert to VHD and file back to VHD
image_utils.fetch_to_vhd(context, image_service, image_id,
self.local_path(volume))
self.local_path(volume),
self.configuration.volume_dd_blocksize)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""

1
cinder/volume/drivers/xenapi/sm.py
View File

@ -176,6 +176,7 @@ class XenAPINFSDriver(driver.VolumeDriver):
image_service,
image_id,
device,
self.configuration.volume_dd_blocksize,
size=volume['size'])
def _use_glance_plugin_to_copy_image_to_volume(self, context, volume,

16
cinder/volume/utils.py
View File

@ -31,15 +31,7 @@ from cinder import units
from cinder import utils
volume_opts = [
cfg.StrOpt('volume_dd_blocksize',
default='1M',
help='The default block size used when copying/clearing '
'volumes'),
]
CONF = cfg.CONF
CONF.register_opts(volume_opts)
LOG = logging.getLogger(__name__)
@ -141,8 +133,8 @@ def notify_about_snapshot_usage(context, snapshot, event_suffix,
notifier_api.INFO, usage_info)
def _calculate_count(size_in_m):
blocksize = CONF.volume_dd_blocksize
def _calculate_count(size_in_m, blocksize):
# Check if volume_dd_blocksize is valid
try:
# Rule out zero-sized/negative dd blocksize which
@ -166,7 +158,7 @@ def _calculate_count(size_in_m):
return blocksize, int(count)
def copy_volume(srcstr, deststr, size_in_m, sync=False,
def copy_volume(srcstr, deststr, size_in_m, blocksize, sync=False,
execute=utils.execute):
# Use O_DIRECT to avoid thrashing the system buffer cache
extra_flags = ['iflag=direct', 'oflag=direct']
@ -184,7 +176,7 @@ def copy_volume(srcstr, deststr, size_in_m, sync=False,
if sync and not extra_flags:
extra_flags.append('conv=fdatasync')
blocksize, count = _calculate_count(size_in_m)
blocksize, count = _calculate_count(size_in_m, blocksize)
# Perform the copy
execute('dd', 'if=%s' % srcstr, 'of=%s' % deststr,

13
etc/cinder/cinder.conf.sample
View File

@ -1110,6 +1110,10 @@
# will autodetect type of backing device (string value)
#iscsi_iotype=fileio
# The default block size used when copying/clearing volumes
# (string value)
#volume_dd_blocksize=1M
#
# Options defined in cinder.volume.drivers.block_device
@ -1831,15 +1835,6 @@
#volume_service_inithost_offload=false
#
# Options defined in cinder.volume.utils
#
# The default block size used when copying/clearing volumes
# (string value)
#volume_dd_blocksize=1M
[keystone_authtoken]
#