openstack
/
cinder
Code Issues Proposed changes

Merge "Check if volume node has enough space for image operations"

This commit is contained in:
Jenkins 2017-01-27 09:09:15 +00:00 committed by Gerrit Code Review
parent 0b49ee9aa2 494264e797
commit 6932b2b517
8 changed files with 118 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cinder/image/image_utils.py
View File

@ -39,6 +39,7 @@ from oslo_utils import fileutils
from oslo_utils import imageutils
from oslo_utils import timeutils
from oslo_utils import units
import psutil
from cinder import exception
from cinder.i18n import _, _LE, _LI, _LW
@ -351,6 +352,10 @@ def fetch_to_volume_format(context, image_service,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s")
% {'fmt': fmt, 'backing_file': backing_file, })
# NOTE(e0ne): check for free space in destination directory before
# image convertion.
check_available_space(dest, virt_size, image_id)
# NOTE(jdg): I'm using qemu-img convert to write
# to the volume regardless if it *needs* conversion or not
# TODO(avishay): We can speed this up by checking if the image is raw
@ -447,6 +452,17 @@ def check_virtual_size(virtual_size, volume_size, image_id):
return virtual_size
def check_available_space(dest, image_size, image_id):
# TODO(e0ne): replace psutil with shutil.disk_usage when we drop
# Python 2.7 support.
free_space = psutil.disk_usage(dest).free
if free_space <= image_size:
msg = ('There is no space to convert image. '
'Requested: %(image_size), available: %(free_space)'
) % {'image_size': image_size, 'free_space': free_space}
raise exception.ImageUnacceptable(image_id=image_id, reason=msg)
def is_xenserver_format(image_meta):
return (
image_meta['disk_format'] == 'vhd'

1
cinder/tests/unit/image/fake.py
View File

@ -97,6 +97,7 @@ class _FakeImageService(object):
'updated_at': timestamp,
'deleted_at': None,
'deleted': False,
'size': 1024,
'status': 'active',
'visibility': 'public',
'protected': True,

71
cinder/tests/unit/test_image_utils.py
View File

@ -327,10 +327,12 @@ class TestVerifyImage(test.TestCase):
(mock_fileutils.remove_path_on_error.return_value.__exit__
.assert_called_once_with(None, None, None))
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch('cinder.image.image_utils.qemu_img_info')
@mock.patch('cinder.image.image_utils.fileutils')
@mock.patch('cinder.image.image_utils.fetch')
def test_kwargs(self, mock_fetch, mock_fileutils, mock_info):
def test_kwargs(self, mock_fetch, mock_fileutils, mock_info,
mock_check_space):
ctxt = mock.sentinel.context
image_service = mock.Mock()
image_id = mock.sentinel.image_id
@ -591,8 +593,9 @@ class TestFetchToVhd(test.TestCase):
dest, 'vpc', blocksize, None,
None, run_as_root=True)
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch('cinder.image.image_utils.fetch_to_volume_format')
def test_kwargs(self, mock_fetch_to):
def test_kwargs(self, mock_fetch_to, mock_check_space):
ctxt = mock.sentinel.context
image_service = mock.sentinel.image_service
image_id = mock.sentinel.image_id
@ -629,8 +632,9 @@ class TestFetchToRaw(test.TestCase):
dest, 'raw', blocksize, None,
None, None, run_as_root=True)
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch('cinder.image.image_utils.fetch_to_volume_format')
def test_kwargs(self, mock_fetch_to):
def test_kwargs(self, mock_fetch_to, mock_check_space):
ctxt = mock.sentinel.context
image_service = mock.sentinel.image_service
image_id = mock.sentinel.image_id
@ -653,6 +657,7 @@ class TestFetchToRaw(test.TestCase):
class TestFetchToVolumeFormat(test.TestCase):
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch('cinder.image.image_utils.convert_image')
@mock.patch('cinder.image.image_utils.volume_utils.copy_volume')
@mock.patch(
@ -664,7 +669,8 @@ class TestFetchToVolumeFormat(test.TestCase):
@mock.patch('cinder.image.image_utils.temporary_file')
@mock.patch('cinder.image.image_utils.CONF')
def test_defaults(self, mock_conf, mock_temp, mock_info, mock_fetch,
mock_is_xen, mock_repl_xen, mock_copy, mock_convert):
mock_is_xen, mock_repl_xen, mock_copy, mock_convert,
mock_check_space):
ctxt = mock.sentinel.context
ctxt.user_id = mock.sentinel.user_id
image_service = mock.Mock(temp_images=None)
@ -697,6 +703,7 @@ class TestFetchToVolumeFormat(test.TestCase):
mock_convert.assert_called_once_with(tmp, dest, volume_format,
run_as_root=True)
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch('cinder.image.image_utils.convert_image')
@mock.patch('cinder.image.image_utils.volume_utils.copy_volume')
@mock.patch(
@ -708,7 +715,8 @@ class TestFetchToVolumeFormat(test.TestCase):
@mock.patch('cinder.image.image_utils.temporary_file')
@mock.patch('cinder.image.image_utils.CONF')
def test_kwargs(self, mock_conf, mock_temp, mock_info, mock_fetch,
mock_is_xen, mock_repl_xen, mock_copy, mock_convert):
mock_is_xen, mock_repl_xen, mock_copy, mock_convert,
mock_check_space):
ctxt = mock.sentinel.context
image_service = mock.Mock(temp_images=None)
image_id = mock.sentinel.image_id
@ -745,6 +753,7 @@ class TestFetchToVolumeFormat(test.TestCase):
mock_convert.assert_called_once_with(tmp, dest, volume_format,
run_as_root=run_as_root)
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch('cinder.image.image_utils.convert_image')
@mock.patch('cinder.image.image_utils.volume_utils.copy_volume')
@mock.patch(
@ -757,7 +766,7 @@ class TestFetchToVolumeFormat(test.TestCase):
@mock.patch('cinder.image.image_utils.CONF')
def test_temporary_images(self, mock_conf, mock_temp, mock_info,
mock_fetch, mock_is_xen, mock_repl_xen,
mock_copy, mock_convert):
mock_copy, mock_convert, mock_check_space):
ctxt = mock.sentinel.context
ctxt.user_id = mock.sentinel.user_id
image_service = mock.Mock(temp_images=None)
@ -976,6 +985,51 @@ class TestFetchToVolumeFormat(test.TestCase):
self.assertFalse(mock_copy.called)
self.assertFalse(mock_convert.called)
@mock.patch('psutil.disk_usage')
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch('cinder.image.image_utils.convert_image')
@mock.patch('cinder.image.image_utils.volume_utils.copy_volume')
@mock.patch(
'cinder.image.image_utils.replace_xenserver_image_with_coalesced_vhd')
@mock.patch('cinder.image.image_utils.is_xenserver_format',
return_value=False)
@mock.patch('cinder.image.image_utils.fetch')
@mock.patch('cinder.image.image_utils.qemu_img_info')
@mock.patch('cinder.image.image_utils.temporary_file')
@mock.patch('cinder.image.image_utils.CONF')
def test_check_no_available_space_error(self, mock_conf, mock_temp,
mock_info, mock_fetch, mock_is_xen,
mock_repl_xen, mock_copy,
mock_convert, mock_check_space,
mock_disk_usage):
ctxt = mock.sentinel.context
image_service = mock.Mock(temp_images=None)
image_id = mock.sentinel.image_id
dest = mock.sentinel.dest
volume_format = mock.sentinel.volume_format
blocksize = mock.sentinel.blocksize
ctxt.user_id = user_id = mock.sentinel.user_id
project_id = mock.sentinel.project_id
size = 1234
run_as_root = mock.sentinel.run_as_root
mock_disk_usage.return_value = units.Gi - 1
data = mock_info.return_value
data.file_format = volume_format
data.backing_file = None
data.virtual_size = units.Gi
mock_check_space.side_effect = exception.ImageUnacceptable(
image_id='fake_image_id', reason='test')
self.assertRaises(
exception.ImageUnacceptable,
image_utils.fetch_to_volume_format,
ctxt, image_service, image_id, dest, volume_format, blocksize,
user_id=user_id, project_id=project_id, size=size,
run_as_root=run_as_root)
@mock.patch('cinder.image.image_utils.convert_image')
@mock.patch('cinder.image.image_utils.volume_utils.copy_volume')
@mock.patch(
@ -1072,6 +1126,7 @@ class TestFetchToVolumeFormat(test.TestCase):
self.assertFalse(mock_copy.called)
self.assertFalse(mock_convert.called)
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch('cinder.image.image_utils.convert_image')
@mock.patch('cinder.image.image_utils.volume_utils.copy_volume')
@mock.patch(
@ -1085,6 +1140,7 @@ class TestFetchToVolumeFormat(test.TestCase):
def _test_format_name_mismatch(self, mock_conf, mock_temp, mock_info,
mock_fetch, mock_is_xen, mock_repl_xen,
mock_copy, mock_convert,
mock_check_space,
legacy_format_name=False):
ctxt = mock.sentinel.context
image_service = mock.Mock(temp_images=None)
@ -1137,6 +1193,7 @@ class TestFetchToVolumeFormat(test.TestCase):
# the legacy 'vpc' format name if 'vhd' is requested.
self._test_format_name_mismatch(legacy_format_name=True)
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch('cinder.image.image_utils.convert_image')
@mock.patch('cinder.image.image_utils.volume_utils.copy_volume')
@mock.patch(
@ -1149,7 +1206,7 @@ class TestFetchToVolumeFormat(test.TestCase):
@mock.patch('cinder.image.image_utils.CONF')
def test_xenserver_to_vhd(self, mock_conf, mock_temp, mock_info,
mock_fetch, mock_is_xen, mock_repl_xen,
mock_copy, mock_convert):
mock_copy, mock_convert, mock_check_space):
ctxt = mock.sentinel.context
image_service = mock.Mock(temp_images=None)
image_id = mock.sentinel.image_id

3
cinder/tests/unit/volume/__init__.py
View File

@ -69,6 +69,8 @@ class BaseVolumeTestCase(test.TestCase):
fake_image.mock_image_service(self)
self.mock_object(brick_lvm.LVM, '_vg_exists', lambda x: True)
self.mock_object(os.path, 'exists', lambda x: True)
self.mock_object(image_utils, 'check_available_space',
lambda x, y, z: True)
self.volume.driver.set_initialized()
self.volume.stats = {'allocated_capacity_gb': 0,
'pools': {}}
@ -140,6 +142,7 @@ class BaseVolumeTestCase(test.TestCase):
request_spec = {
'volume_properties': self.volume_params,
'image_id': image_id,
'image_size': 1
}
self.volume.create_volume(self.context, volume, request_spec)
finally:

11
cinder/tests/unit/volume/drivers/test_rbd.py
View File

@ -1660,8 +1660,10 @@ class ManagedRBDTestCase(test_volume.DriverTestCase):
# cleanup
volume.destroy()
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch.object(cinder.image.glance, 'get_default_image_service')
def test_create_vol_from_image_status_available(self, mock_gdis):
def test_create_vol_from_image_status_available(self, mock_gdis,
mock_check_space):
"""Clone raw image then verify volume is in available state."""
def _mock_clone_image(context, volume, image_location,
@ -1682,11 +1684,12 @@ class ManagedRBDTestCase(test_volume.DriverTestCase):
self.assertFalse(mock_create.called)
self.assertTrue(mock_gdis.called)
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch.object(cinder.image.glance, 'get_default_image_service')
@mock.patch('cinder.image.image_utils.TemporaryImages.fetch')
@mock.patch('cinder.image.image_utils.qemu_img_info')
def test_create_vol_from_non_raw_image_status_available(
self, mock_qemu_info, mock_fetch, mock_gdis):
self, mock_qemu_info, mock_fetch, mock_gdis, mock_check_space):
"""Clone non-raw image then verify volume is in available state."""
def _mock_clone_image(context, volume, image_location,
@ -1712,8 +1715,10 @@ class ManagedRBDTestCase(test_volume.DriverTestCase):
self.assertTrue(mock_create.called)
self.assertTrue(mock_gdis.called)
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch.object(cinder.image.glance, 'get_default_image_service')
def test_create_vol_from_image_status_error(self, mock_gdis):
def test_create_vol_from_image_status_error(self, mock_gdis,
mock_check_space):
"""Fail to clone raw image then verify volume is in error state."""
with mock.patch.object(self.volume.driver, 'clone_image') as \
mock_clone_image:

27
cinder/tests/unit/volume/flows/test_create_volume_flow.py
View File

@ -854,6 +854,7 @@ class CreateVolumeFlowManagerGlanceCinderBackendCase(test.TestCase):
image_meta = {'id': image_id,
'container_format': 'bare',
'disk_format': format,
'size': 1024,
'owner': owner or self.ctxt.project_id,
'virtual_size': None}
@ -921,7 +922,7 @@ class CreateVolumeFlowManagerImageCacheTestCase(test.TestCase):
image_location = 'someImageLocationStr'
image_id = fakes.IMAGE_ID
image_meta = {'virtual_size': '1073741824'}
image_meta = {'virtual_size': '1073741824', 'size': 1073741824}
manager = create_volume_manager.CreateVolumeFromSpecTask(
self.mock_volume_manager,
@ -968,7 +969,7 @@ class CreateVolumeFlowManagerImageCacheTestCase(test.TestCase):
image_location = 'someImageLocationStr'
image_id = fakes.IMAGE_ID
image_meta = {'virtual_size': '1073741824'}
image_meta = {'virtual_size': '1073741824', 'size': 1073741824}
manager = create_volume_manager.CreateVolumeFromSpecTask(
self.mock_volume_manager,
@ -1052,7 +1053,7 @@ class CreateVolumeFlowManagerImageCacheTestCase(test.TestCase):
image_location = 'someImageLocationStr'
image_id = fakes.IMAGE_ID
image_meta = {'virtual_size': '2147483648'}
image_meta = {'virtual_size': '2147483648', 'size': 2147483648}
manager = create_volume_manager.CreateVolumeFromSpecTask(
self.mock_volume_manager,
@ -1085,7 +1086,7 @@ class CreateVolumeFlowManagerImageCacheTestCase(test.TestCase):
image_location = 'someImageLocationStr'
image_id = fakes.IMAGE_ID
image_meta = {'virtual_size': None}
image_meta = {'virtual_size': None, 'size': 1024}
manager = create_volume_manager.CreateVolumeFromSpecTask(
self.mock_volume_manager,
@ -1122,10 +1123,12 @@ class CreateVolumeFlowManagerImageCacheTestCase(test.TestCase):
@mock.patch('cinder.db.volume_update')
@mock.patch('cinder.objects.Volume.get_by_id')
@mock.patch('cinder.image.image_utils.qemu_img_info')
@mock.patch('cinder.image.image_utils.check_available_space')
def test_create_from_image_cache_miss(
self, mock_qemu_info, mock_volume_get, mock_volume_update,
mock_get_internal_context, mock_create_from_img_dl,
mock_create_from_src, mock_handle_bootable, mock_fetch_img):
self, mock_check_size, mock_qemu_info, mock_volume_get,
mock_volume_update, mock_get_internal_context,
mock_create_from_img_dl, mock_create_from_src,
mock_handle_bootable, mock_fetch_img):
mock_get_internal_context.return_value = self.ctxt
mock_fetch_img.return_value = mock.MagicMock(
spec=utils.get_file_spec())
@ -1189,9 +1192,10 @@ class CreateVolumeFlowManagerImageCacheTestCase(test.TestCase):
@mock.patch('cinder.db.volume_update')
@mock.patch('cinder.objects.Volume.get_by_id')
@mock.patch('cinder.image.image_utils.qemu_img_info')
@mock.patch('cinder.image.image_utils.check_available_space')
def test_create_from_image_cache_miss_error_downloading(
self, mock_qemu_info, mock_volume_get, mock_volume_update,
mock_get_internal_context,
self, mock_check_size, mock_qemu_info, mock_volume_get,
mock_volume_update, mock_get_internal_context,
mock_create_from_img_dl, mock_create_from_src,
mock_handle_bootable, mock_fetch_img):
mock_fetch_img.return_value = mock.MagicMock()
@ -1267,7 +1271,7 @@ class CreateVolumeFlowManagerImageCacheTestCase(test.TestCase):
image_location = 'someImageLocationStr'
image_id = fakes.IMAGE_ID
image_meta = {'virtual_size': '1073741824'}
image_meta = {'virtual_size': '1073741824', 'size': 1073741824}
manager = create_volume_manager.CreateVolumeFromSpecTask(
self.mock_volume_manager,
@ -1313,9 +1317,10 @@ class CreateVolumeFlowManagerImageCacheTestCase(test.TestCase):
image_meta=image_meta
)
@mock.patch('cinder.image.image_utils.check_available_space')
@mock.patch('cinder.image.image_utils.qemu_img_info')
def test_create_from_image_cache_miss_error_size_invalid(
self, mock_qemu_info, mock_get_internal_context,
self, mock_qemu_info, mock_check_space, mock_get_internal_context,
mock_create_from_img_dl, mock_create_from_src,
mock_handle_bootable, mock_fetch_img):
mock_fetch_img.return_value = mock.MagicMock()

9
cinder/volume/flows/manager/create_volume.py
View File

@ -10,6 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
import traceback
from oslo_concurrency import processutils
@ -713,6 +714,14 @@ class CreateVolumeFromSpecTask(flow_utils.CinderTask):
{'volume_id': volume.id,
'image_location': image_location, 'image_id': image_id})
# NOTE(e0ne): check for free space in image_conversion_dir before
# image downloading.
if (CONF.image_conversion_dir and not
os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
image_utils.check_available_space(CONF.image_conversion_dir,
image_meta['size'], image_id)
virtual_size = image_meta.get('virtual_size')
if virtual_size:
virtual_size = image_utils.check_virtual_size(virtual_size,

1
requirements.txt
View File

@ -34,6 +34,7 @@ osprofiler>=1.4.0 # Apache-2.0
paramiko>=2.0 # LGPLv2.1+
Paste # MIT
PasteDeploy>=1.5.0 # MIT
psutil>=3.0.1 # BSD
pycrypto>=2.6 # Public Domain
pyparsing>=2.0.7 # MIT
python-barbicanclient>=4.0.0 # Apache-2.0