From 1e74f318d68c131948dcad566ac11034bcef49ab Mon Sep 17 00:00:00 2001 From: Eric Harney Date: Wed, 3 Jul 2019 11:25:26 -0400 Subject: [PATCH] Compress images uploaded to Glance When possible (based on the image format), compress images that are uploaded to Glance. For now, this is done for the qcow2 format, but can be expanded to other formats as needed. This should be a performance win for some deployments, consuming less network/storage at the expense of CPU. An "image_compress_on_upload" option is provided to disable this behavior for deployments that want to prioritize minimal CPU usage, etc. Closes-Bug: #1824821 Change-Id: I22bd2be6653bdc4e755ea8a9d8780e8e67b4485a --- cinder/image/image_utils.py | 45 +++++++++++++++---- cinder/tests/unit/test_image_utils.py | 32 ++++++++++++- cinder/volume/driver.py | 3 +- .../compress-images-fed3e354d94b0845.yaml | 7 +++ 4 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 releasenotes/notes/compress-images-fed3e354d94b0845.yaml diff --git a/cinder/image/image_utils.py b/cinder/image/image_utils.py index 7dd4ee059ef..93eb850a59d 100644 --- a/cinder/image/image_utils.py +++ b/cinder/image/image_utils.py @@ -57,7 +57,12 @@ image_opts = [ cfg.StrOpt('image_conversion_dir', default='$state_path/conversion', help='Directory used for temporary storage ' - 'during image conversion'), ] + 'during image conversion'), + cfg.BoolOpt('image_compress_on_upload', + default=True, + help='When possible, compress images uploaded ' + 'to the image service'), +] CONF = cfg.CONF CONF.register_opts(image_opts) @@ -79,6 +84,8 @@ QEMU_IMG_VERSION = None QEMU_IMG_MIN_FORCE_SHARE_VERSION = [2, 10, 0] QEMU_IMG_MIN_CONVERT_LUKS_VERSION = '2.10' +COMPRESSIBLE_IMAGE_FORMATS = ('qcow2') + def fixup_disk_format(disk_format): """Return the format to be provided to qemu-img convert.""" @@ -141,7 +148,8 @@ def qemu_img_supports_force_share(): def _get_qemu_convert_cmd(src, dest, out_format, src_format=None, out_subformat=None, cache_mode=None, - prefix=None, cipher_spec=None, passphrase_file=None): + prefix=None, cipher_spec=None, + passphrase_file=None, compress=False): if out_format == 'vhd': # qemu-img still uses the legacy vpc name @@ -155,6 +163,10 @@ def _get_qemu_convert_cmd(src, dest, out_format, src_format=None, if cache_mode: cmd += ('-t', cache_mode) + if CONF.image_compress_on_upload and compress: + if out_format in COMPRESSIBLE_IMAGE_FORMATS: + cmd += ('-c',) + if out_subformat: cmd += ('-o', 'subformat=%s' % out_subformat) @@ -206,8 +218,21 @@ def check_qemu_img_version(minimum_version): def _convert_image(prefix, source, dest, out_format, out_subformat=None, src_format=None, - run_as_root=True, cipher_spec=None, passphrase_file=None): - """Convert image to other format.""" + run_as_root=True, cipher_spec=None, + passphrase_file=None, compress=False): + """Convert image to other format. + + :param prefix: command prefix, i.e. cgexec for throttling + :param source: source filename + :param dest: destination filename + :param out_format: output image format of qemu-img + :param out_subformat: output image subformat + :param src_format: source image format + :param run_as_root: run qemu-img as root + :param cipher_spec: encryption details + :param passphrase_file: filename containing luks passphrase + :param compress: compress w/ qemu-img when possible (best effort) + """ # Check whether O_DIRECT is supported and set '-t none' if it is # This is needed to ensure that all data hit the device before @@ -234,7 +259,8 @@ def _convert_image(prefix, source, dest, out_format, cache_mode=cache_mode, prefix=prefix, cipher_spec=cipher_spec, - passphrase_file=passphrase_file) + passphrase_file=passphrase_file, + compress=compress) start_time = timeutils.utcnow() @@ -286,7 +312,8 @@ def _convert_image(prefix, source, dest, out_format, def convert_image(source, dest, out_format, out_subformat=None, src_format=None, run_as_root=True, throttle=None, - cipher_spec=None, passphrase_file=None): + cipher_spec=None, passphrase_file=None, + compress=False): if not throttle: throttle = throttling.Throttle.get_default() with throttle.subcommand(source, dest) as throttle_cmd: @@ -297,7 +324,8 @@ def convert_image(source, dest, out_format, out_subformat=None, src_format=src_format, run_as_root=run_as_root, cipher_spec=cipher_spec, - passphrase_file=passphrase_file) + passphrase_file=passphrase_file, + compress=compress) def resize_image(source, size, run_as_root=False): @@ -641,7 +669,8 @@ def upload_volume(context, image_service, image_meta, volume_path, out_format = fixup_disk_format(image_meta['disk_format']) convert_image(volume_path, tmp, out_format, - run_as_root=run_as_root) + run_as_root=run_as_root, + compress=True) data = qemu_img_info(tmp, run_as_root=run_as_root) if data.file_format != out_format: diff --git a/cinder/tests/unit/test_image_utils.py b/cinder/tests/unit/test_image_utils.py index 1f5df668cbf..f994b259f53 100644 --- a/cinder/tests/unit/test_image_utils.py +++ b/cinder/tests/unit/test_image_utils.py @@ -124,6 +124,7 @@ class TestQemuImgInfo(test.TestCase): current_version=[1, 8]) +@ddt.ddt class TestConvertImage(test.TestCase): @mock.patch('cinder.image.image_utils.qemu_img_info') @mock.patch('cinder.utils.execute') @@ -280,6 +281,31 @@ class TestConvertImage(test.TestCase): '-O', 'vpc', source, dest, run_as_root=True) + @ddt.data(True, False) + @mock.patch('cinder.image.image_utils.qemu_img_info') + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.utils.is_blk_device', return_value=False) + def test_convert_to_qcow2(self, + compress_option, + mock_isblk, mock_exec, mock_info): + self.override_config('image_compress_on_upload', compress_option) + source = mock.sentinel.source + dest = mock.sentinel.dest + out_format = 'qcow2' + mock_info.return_value.virtual_size = 1048576 + + image_utils.convert_image(source, + dest, + out_format, + compress=True) + + exec_args = ['qemu-img', 'convert', '-O', 'qcow2'] + if compress_option: + exec_args.append('-c') + exec_args.extend((source, dest)) + mock_exec.assert_called_once_with(*exec_args, + run_as_root=True) + @mock.patch('cinder.image.image_utils.CONF') @mock.patch('cinder.volume.utils.check_for_odirect_support', return_value=True) @@ -727,7 +753,8 @@ class TestUploadVolume(test.TestCase): mock_convert.assert_called_once_with(volume_path, temp_file, output_format, - run_as_root=True) + run_as_root=True, + compress=True) mock_info.assert_called_with(temp_file, run_as_root=True) self.assertEqual(2, mock_info.call_count) mock_open.assert_called_once_with(temp_file, 'rb') @@ -823,7 +850,8 @@ class TestUploadVolume(test.TestCase): mock_convert.assert_called_once_with(volume_path, temp_file, mock.sentinel.disk_format, - run_as_root=True) + run_as_root=True, + compress=True) mock_info.assert_called_with(temp_file, run_as_root=True) self.assertEqual(2, mock_info.call_count) self.assertFalse(image_service.update.called) diff --git a/cinder/volume/driver.py b/cinder/volume/driver.py index 6125c53c8e6..bdd979d718d 100644 --- a/cinder/volume/driver.py +++ b/cinder/volume/driver.py @@ -896,7 +896,8 @@ class BaseVD(object): image_utils.upload_volume(context, image_service, image_meta, - attach_info['device']['path']) + attach_info['device']['path'], + compress=True) finally: # Since attached volume was not used for writing we can force # detach it diff --git a/releasenotes/notes/compress-images-fed3e354d94b0845.yaml b/releasenotes/notes/compress-images-fed3e354d94b0845.yaml new file mode 100644 index 00000000000..8b8f2c0cc67 --- /dev/null +++ b/releasenotes/notes/compress-images-fed3e354d94b0845.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + When uploading qcow2 images to Glance, image data will be compressed. This + will generally result in less data transferred to Glance at the expense of + higher CPU usage. This behavior is controlled by the + "image_compress_on_upload" boolean option, which defaults to True.