diff --git a/cinder/common/config.py b/cinder/common/config.py index 79790c909f3..767f23df7d6 100644 --- a/cinder/common/config.py +++ b/cinder/common/config.py @@ -195,12 +195,24 @@ image_opts = [ 'value is used.'), ] +compression_opts = [ + cfg.StrOpt('compression_format', + default='gzip', + choices=[('gzip', 'GNUzip format')], + help='Image compression format on image upload'), + cfg.BoolOpt('allow_compression_on_image_upload', + default=False, + help='The strategy to use for image compression on upload. ' + 'Default is disallow compression.'), +] + CONF.register_opts(api_opts) CONF.register_opts(core_opts) CONF.register_opts(auth_opts) CONF.register_opts(backup_opts) CONF.register_opts(image_opts) CONF.register_opts(global_opts) +CONF.register_opts(compression_opts) def set_middleware_defaults(): diff --git a/cinder/exception.py b/cinder/exception.py index 1638ca499ca..2396b8ca722 100644 --- a/cinder/exception.py +++ b/cinder/exception.py @@ -1099,3 +1099,16 @@ class ServiceUserTokenNoAuth(CinderException): class RekeyNotSupported(CinderException): message = _("Rekey not supported.") + + +class ImageCompressionNotAllowed(CinderException): + message = _("Image compression upload disallowed, but container_format " + "is compressed") + + +class CinderAcceleratorError(CinderException): + message = _("Cinder accelerator %(accelerator)s encountered an error " + "while compressing/decompressing image.\n" + "Command %(cmd)s execution failed.\n" + "%(description)s\n" + "Reason: %(reason)s") diff --git a/cinder/image/accelerator.py b/cinder/image/accelerator.py new file mode 100644 index 00000000000..a471fb508d1 --- /dev/null +++ b/cinder/image/accelerator.py @@ -0,0 +1,105 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc + +from oslo_config import cfg +from oslo_log import log as logging +from oslo_utils import importutils +import six + +from cinder import exception +from cinder.i18n import _ + +LOG = logging.getLogger(__name__) + +CONF = cfg.CONF + +# NOTE(ZhengMa): The order of the option is improtant, accelerators +# are looked by this list order +# Be careful to edit it +_ACCEL_PATH_PREFERENCE_ORDER_LIST = [ + 'cinder.image.accelerators.qat.AccelQAT', + 'cinder.image.accelerators.gzip.AccelGZIP', +] + + +@six.add_metaclass(abc.ABCMeta) +class AccelBase(object): + def __init__(self): + return + + @abc.abstractmethod + def is_accel_exist(self): + return + + @abc.abstractmethod + def compress_img(self, run_as_root): + return + + @abc.abstractmethod + def decompress_img(self, run_as_root): + return + + +class ImageAccel(object): + + def __init__(self, src, dest): + self.src = src + self.dest = dest + self.compression_format = CONF.compression_format + if(self.compression_format == 'gzip'): + self._accel_engine_path = _ACCEL_PATH_PREFERENCE_ORDER_LIST + else: + self._accel_engine_path = None + self.engine = self._get_engine() + + def _get_engine(self, *args, **kwargs): + if self._accel_engine_path: + for accel in self._accel_engine_path: + engine_cls = importutils.import_class(accel) + eng = engine_cls(*args, **kwargs) + if eng.is_accel_exist(): + return eng + + ex_msg = _("No valid accelerator") + raise exception.CinderException(ex_msg) + + def is_engine_ready(self): + + if not self.engine: + return False + if not self.engine.is_accel_exist(): + return False + return True + + def compress_img(self, run_as_root): + if not self.is_engine_ready(): + return + self.engine.compress_img(self.src, + self.dest, + run_as_root) + + def decompress_img(self, run_as_root): + if not self.is_engine_ready(): + return + self.engine.decompress_img(self.src, + self.dest, + run_as_root) + + +def is_gzip_compressed(image_file): + # The first two bytes of a gzip file are: 1f 8b + GZIP_MAGIC_BYTES = b'\x1f\x8b' + with open(image_file, 'rb') as f: + return f.read(2) == GZIP_MAGIC_BYTES diff --git a/cinder/image/accelerators/__init__.py b/cinder/image/accelerators/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cinder/image/accelerators/gzip.py b/cinder/image/accelerators/gzip.py new file mode 100644 index 00000000000..1b92cb5b22a --- /dev/null +++ b/cinder/image/accelerators/gzip.py @@ -0,0 +1,98 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from oslo_concurrency import processutils +from oslo_log import log as logging + +from cinder import exception +from cinder.i18n import _ +from cinder.image import accelerator +from cinder import utils + +LOG = logging.getLogger(__name__) + + +class AccelGZIP(accelerator.AccelBase): + def is_accel_exist(self): + cmd = ['which', 'gzip'] + try: + utils.execute(*cmd) + except processutils.ProcessExecutionError: + LOG.error("GZIP package is not installed.") + return False + + return True + + # NOTE(ZhengMa): Gzip compresses a file in-place and adds a .gz + # extension to the filename, so we rename the compressed file back + # to the name Cinder expects it to have. + # (Cinder expects to have A to upload) + # Follow these steps: + # 1. compress A to A.gz (gzip_out_file is A.gz) + # 2. mv A.gz to A (gzip_out_file to dest) + def compress_img(self, src, dest, run_as_root): + try: + gzip_compress_cmd = ['gzip', '-k', src] + utils.execute(*gzip_compress_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + raise exception.CinderAcceleratorError( + accelerator='GZIP', + description=_("Volume compression failed while " + "uploading to glance. GZIP compression " + "command failed."), + cmd=gzip_compress_cmd, + reason=ex.stderr) + try: + gzip_output_filename = src + '.gz' + mv_cmd = ['mv', gzip_output_filename, dest] + utils.execute(*mv_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + fnames = {'i_fname': gzip_output_filename, 'o_fname': dest} + raise exception.CinderAcceleratorError( + accelerator='GZIP', + description = _("Failed to rename %(i_fname)s " + "to %(o_fname)s") % fnames, + cmd=mv_cmd, + reason=ex.stderr) + + # NOTE(ZhengMa): Gzip can only decompresses a file with a .gz + # extension to the filename, so we rename the original file so + # that it can be accepted by Gzip. + # Follow these steps: + # 1. mv A to A.gz (gzip_in_file is A.gz) + # 2. decompress A.gz to A (gzip_in_file to dest) + def decompress_img(self, src, dest, run_as_root): + try: + gzip_input_filename = dest + '.gz' + mv_cmd = ['mv', src, gzip_input_filename] + utils.execute(*mv_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + fnames = {'i_fname': src, 'o_fname': gzip_input_filename} + raise exception.CinderAcceleratorError( + accelerator='GZIP', + description = _("Failed to rename %(i_fname)s " + "to %(o_fname)s") % fnames, + cmd=mv_cmd, + reason=ex.stderr) + try: + gzip_decompress_cmd = ['gzip', '-d', gzip_input_filename] + utils.execute(*gzip_decompress_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + raise exception.CinderAcceleratorError( + accelerator='GZIP', + description = _("Image decompression failed while " + "downloading from glance. GZIP " + "decompression command failed."), + cmd=gzip_decompress_cmd, + reason=ex.stderr) diff --git a/cinder/image/accelerators/qat.py b/cinder/image/accelerators/qat.py new file mode 100644 index 00000000000..b07c583fb03 --- /dev/null +++ b/cinder/image/accelerators/qat.py @@ -0,0 +1,99 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from oslo_concurrency import processutils +from oslo_log import log as logging + +from cinder import exception +from cinder.i18n import _ +from cinder.image import accelerator +from cinder import utils + + +LOG = logging.getLogger(__name__) + + +class AccelQAT(accelerator.AccelBase): + def is_accel_exist(self): + cmd = ['which', 'qzip'] + try: + utils.execute(*cmd) + except processutils.ProcessExecutionError: + LOG.error("QATzip package is not installed.") + return False + + return True + + # NOTE(ZhengMa): QATzip compresses a file in-place and adds a .gz + # extension to the filename, so we rename the compressed file back + # to the name Cinder expects it to have. + # (Cinder expects to have A to upload) + # Follow these steps: + # 1. compress A to A.gz (src to qat_out_file) + # 2. mv A.gz to A (qat_out_file to dest) + def compress_img(self, src, dest, run_as_root): + try: + qat_compress_cmd = ['qzip', '-k', src, '-o', dest] + utils.execute(*qat_compress_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + raise exception.CinderAcceleratorError( + accelerator='QAT', + description=_("Volume compression failed while " + "uploading to glance. QAT compression " + "command failed."), + cmd=qat_compress_cmd, + reason=ex.stderr) + try: + qat_output_filename = src + '.gz' + mv_cmd = ['mv', qat_output_filename, dest] + utils.execute(*mv_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + fnames = {'i_fname': qat_output_filename, 'o_fname': dest} + raise exception.CinderAcceleratorError( + accelerator='QAT', + description = _("Failed to rename %(i_fname)s " + "to %(o_fname)s") % fnames, + cmd=mv_cmd, + reason=ex.stderr) + + # NOTE(ZhengMa): QATzip can only decompresses a file with a .gz + # extension to the filename, so we rename the original file so + # that it can be accepted by QATzip. + # Follow these steps: + # 1. mv A to A.gz (qat_in_file is A.gz) + # 2. decompress A.gz to A (qat_in_file to dest) + def decompress_img(self, src, dest, run_as_root): + try: + qat_input_filename = dest + '.gz' + mv_cmd = ['mv', src, qat_input_filename] + utils.execute(*mv_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + fnames = {'i_fname': src, 'o_fname': qat_input_filename} + raise exception.CinderAcceleratorError( + accelerator='QAT', + description = _("Failed to rename %(i_fname)s " + "to %(o_fname)s") % fnames, + cmd=mv_cmd, + reason=ex.stderr) + try: + qat_decompress_cmd = ['qzip', '-d', qat_input_filename] + utils.execute(*qat_decompress_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + raise exception.CinderAcceleratorError( + accelerator='QAT', + description = _("Image decompression failed while " + "downloading from glance. QAT " + "decompression command failed."), + cmd=qat_decompress_cmd, + reason=ex.stderr) diff --git a/cinder/image/image_utils.py b/cinder/image/image_utils.py index 7d57b839ebf..93c66a4be26 100644 --- a/cinder/image/image_utils.py +++ b/cinder/image/image_utils.py @@ -47,6 +47,7 @@ import six from cinder import exception from cinder.i18n import _ +from cinder.image import accelerator from cinder import utils from cinder.volume import throttling from cinder.volume import volume_utils @@ -279,7 +280,6 @@ def _convert_image(prefix, source, dest, out_format, LOG.error(message) raise - duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in @@ -551,6 +551,17 @@ def fetch_to_volume_format(context, image_service, qemu_img = True image_meta = image_service.show(context, image_id) + allow_image_compression = CONF.allow_compression_on_image_upload + if image_meta and (image_meta.get('container_format') == 'compressed'): + if allow_image_compression is False: + compression_param = {'container_format': + image_meta.get('container_format')} + raise exception.ImageUnacceptable( + image_id=image_id, + reason=_("Image compression disallowed, " + "but container_format is " + "%(container_format)s.") % compression_param) + # NOTE(avishay): I'm not crazy about creating temp files which may be # large and cause disk full errors which would confuse users. # Unfortunately it seems that you can't pipe to 'qemu-img convert' because @@ -608,6 +619,21 @@ def fetch_to_volume_format(context, image_service, reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % {'fmt': fmt, 'backing_file': backing_file, }) + # NOTE(ZhengMa): This is used to do image decompression on image + # downloading with 'compressed' container_format. It is a + # transparent level between original image downloaded from + # Glance and Cinder image service. So the source file path is + # the same with destination file path. + if image_meta.get('container_format') == 'compressed': + LOG.debug("Found image with compressed container format") + if not accelerator.is_gzip_compressed(tmp): + raise exception.ImageUnacceptable( + image_id=image_id, + reason=_("Unsupported compressed image format found. " + "Only gzip is supported currently")) + accel = accelerator.ImageAccel(tmp, tmp) + accel.decompress_img(run_as_root=run_as_root) + # NOTE(jdg): I'm using qemu-img convert to write # to the volume regardless if it *needs* conversion or not # TODO(avishay): We can speed this up by checking if the image is raw @@ -615,8 +641,8 @@ def fetch_to_volume_format(context, image_service, # check via 'qemu-img info' that what we copied was in fact a raw # image and not a different format with a backing file, which may be # malicious. - LOG.debug("%s was %s, converting to %s ", image_id, fmt, volume_format) disk_format = fixup_disk_format(image_meta['disk_format']) + LOG.debug("%s was %s, converting to %s", image_id, fmt, volume_format) convert_image(tmp, dest, volume_format, out_subformat=volume_subformat, @@ -636,19 +662,20 @@ def _validate_file_format(image_data, expected_format): def upload_volume(context, image_service, image_meta, volume_path, volume_format='raw', run_as_root=True, compress=True): image_id = image_meta['id'] - if (image_meta['disk_format'] == volume_format): - LOG.debug("%s was %s, no need to convert to %s", - image_id, volume_format, image_meta['disk_format']) - if os.name == 'nt' or os.access(volume_path, os.R_OK): - with open(volume_path, 'rb') as image_file: - image_service.update(context, image_id, {}, - tpool.Proxy(image_file)) - else: - with utils.temporary_chown(volume_path): + if image_meta.get('container_format') != 'compressed': + if (image_meta['disk_format'] == volume_format): + LOG.debug("%s was %s, no need to convert to %s", + image_id, volume_format, image_meta['disk_format']) + if os.name == 'nt' or os.access(volume_path, os.R_OK): with open(volume_path, 'rb') as image_file: image_service.update(context, image_id, {}, tpool.Proxy(image_file)) - return + else: + with utils.temporary_chown(volume_path): + with open(volume_path, 'rb') as image_file: + image_service.update(context, image_id, {}, + tpool.Proxy(image_file)) + return with temporary_file() as tmp: LOG.debug("%s was %s, converting to %s", @@ -679,6 +706,14 @@ def upload_volume(context, image_service, image_meta, volume_path, reason=_("Converted to %(f1)s, but format is now %(f2)s") % {'f1': out_format, 'f2': data.file_format}) + # NOTE(ZhengMa): This is used to do image compression on image + # uploading with 'compressed' container_format. + # Compress file 'tmp' in-place + if image_meta.get('container_format') == 'compressed': + LOG.debug("Container_format set to 'compressed', compressing " + "image before uploading.") + accel = accelerator.ImageAccel(tmp, tmp) + accel.compress_img(run_as_root=run_as_root) with open(tmp, 'rb') as image_file: image_service.update(context, image_id, {}, tpool.Proxy(image_file)) diff --git a/cinder/opts.py b/cinder/opts.py index 6fbc39cf9a4..a6872c4ba85 100644 --- a/cinder/opts.py +++ b/cinder/opts.py @@ -226,6 +226,7 @@ def list_opts(): cinder_common_config.backup_opts, cinder_common_config.image_opts, cinder_common_config.global_opts, + cinder_common_config.compression_opts, cinder.compute.compute_opts, cinder_context.context_opts, cinder_db_api.db_opts, diff --git a/cinder/tests/unit/image/accelerators/__init__.py b/cinder/tests/unit/image/accelerators/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cinder/tests/unit/image/accelerators/test_qat_gzip.py b/cinder/tests/unit/image/accelerators/test_qat_gzip.py new file mode 100644 index 00000000000..3fa87a94e85 --- /dev/null +++ b/cinder/tests/unit/image/accelerators/test_qat_gzip.py @@ -0,0 +1,217 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from cinder import exception +from cinder.image import accelerator +from cinder import test + + +class TestAccelerators(test.TestCase): + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = True) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = True) + # Compress test, QAT and GZIP available + def test_compress_img_prefer_qat_when_available(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + + source = 'fake_path' + dest = 'fake_path' + + accel = accelerator.ImageAccel(source, dest) + accel.compress_img(run_as_root=True) + + expected = [ + mock.call('qzip', '-k', dest, '-o', dest, + run_as_root=True), + mock.call('mv', dest + '.gz', dest, + run_as_root=True) + ] + + mock_exec.assert_has_calls(expected) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = False) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = True) + # Compress test, QAT not available but GZIP available + def test_compress_img_qat_accel_not_exist_gzip_exist(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + source = 'fake_path' + dest = 'fake_path' + + accel = accelerator.ImageAccel(source, dest) + accel.compress_img(run_as_root=True) + + not_called = mock.call('qzip', '-k', dest, '-o', dest, + run_as_root=True) + + self.assertNotIn(not_called, mock_exec.call_args_list) + + expected = [ + mock.call('gzip', '-k', dest, + run_as_root=True), + mock.call('mv', dest + '.gz', dest, + run_as_root=True) + ] + mock_exec.assert_has_calls(expected) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = True) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = False) + # Compress test, QAT available but GZIP not available + def test_compress_img_prefer_qat_without_gzip(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + + source = 'fake_path' + dest = 'fake_path' + + accel = accelerator.ImageAccel(source, dest) + accel.compress_img(run_as_root=True) + + expected = [ + mock.call('qzip', '-k', dest, '-o', dest, + run_as_root=True), + mock.call('mv', dest + '.gz', dest, + run_as_root=True) + ] + + mock_exec.assert_has_calls(expected) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = False) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = False) + # Compress test, no accelerator available + def test_compress_img_no_accel_exist(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + source = 'fake_path' + dest = 'fake_path' + + self.assertRaises(exception.CinderException, + accelerator.ImageAccel, + source, + dest) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = True) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = True) + # Decompress test, QAT and GZIP available + def test_decompress_img_prefer_qat_when_available(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + source = 'fake_path' + dest = 'fake_path' + + accel = accelerator.ImageAccel(source, dest) + accel.decompress_img(run_as_root=True) + + expected = [ + mock.call('mv', source, source + '.gz', + run_as_root=True), + mock.call('qzip', '-d', source + '.gz', + run_as_root=True) + ] + + mock_exec.assert_has_calls(expected) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = False) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = True) + # Decompress test, QAT not available but GZIP available + def test_decompress_img_qat_accel_not_exist_gzip_exist(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + source = 'fake_path' + dest = 'fake_path' + + accel = accelerator.ImageAccel(source, dest) + accel.decompress_img(run_as_root=True) + + not_called = mock.call('qzip', '-d', source + '.gz', + run_as_root=True) + + self.assertNotIn(not_called, mock_exec.call_args_list) + + expected = [ + mock.call('mv', source, source + '.gz', + run_as_root=True), + mock.call('gzip', '-d', source + '.gz', + run_as_root=True) + ] + + mock_exec.assert_has_calls(expected) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = True) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = False) + # Decompress test, QAT available but GZIP not available + def test_decompress_img_prefer_qat_without_gzip(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + source = 'fake_path' + dest = 'fake_path' + + accel = accelerator.ImageAccel(source, dest) + accel.decompress_img(run_as_root=True) + + expected = [ + mock.call('mv', source, source + '.gz', + run_as_root=True), + mock.call('qzip', '-d', source + '.gz', + run_as_root=True) + ] + + mock_exec.assert_has_calls(expected) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = False) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = False) + # Decompress test, no accelerator available + def test_decompress_img_no_accel_exist(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + source = 'fake_path' + dest = 'fake_path' + + self.assertRaises(exception.CinderException, + accelerator.ImageAccel, + source, + dest) diff --git a/cinder/tests/unit/image/test_accelerator.py b/cinder/tests/unit/image/test_accelerator.py new file mode 100644 index 00000000000..4f7466c7b76 --- /dev/null +++ b/cinder/tests/unit/image/test_accelerator.py @@ -0,0 +1,100 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import mock + +from cinder.image import accelerator +from cinder import test + + +class fakeEngine(object): + + def __init__(self): + pass + + def compress_img(self, src, dest, run_as_root): + pass + + def decompress_img(self, src, dest, run_as_root): + pass + + +class TestAccelerator(test.TestCase): + + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = True) + def test_compress_img_engine_ready(self, mock_accel_engine_ready, + mock_get_engine): + source = mock.sentinel.source + dest = mock.sentinel.dest + run_as_root = mock.sentinel.run_as_root + + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + accel = accelerator.ImageAccel(source, dest) + + accel.compress_img(run_as_root=run_as_root) + mock_engine.compress_img.assert_called() + + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = False) + def test_compress_img_engine_not_ready(self, mock_accel_engine_ready, + mock_get_engine): + + source = mock.sentinel.source + dest = mock.sentinel.dest + run_as_root = mock.sentinel.run_as_root + + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + accel = accelerator.ImageAccel(source, dest) + + accel.compress_img(run_as_root=run_as_root) + mock_engine.compress_img.assert_not_called() + + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = True) + def test_decompress_img_engine_ready(self, mock_accel_engine_ready, + mock_get_engine): + + source = mock.sentinel.source + dest = mock.sentinel.dest + run_as_root = mock.sentinel.run_as_root + + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + accel = accelerator.ImageAccel(source, dest) + + accel.decompress_img(run_as_root=run_as_root) + mock_engine.decompress_img.assert_called() + + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = False) + def test_decompress_img_engine_not_ready(self, mock_accel_engine_ready, + mock_get_engine): + + source = mock.sentinel.source + dest = mock.sentinel.dest + run_as_root = mock.sentinel.run_as_root + + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + accel = accelerator.ImageAccel(source, dest) + + accel.decompress_img(run_as_root=run_as_root) + mock_engine.decompress_img.assert_not_called() diff --git a/cinder/tests/unit/test_image_utils.py b/cinder/tests/unit/test_image_utils.py index cb13dab5ae4..f7040ee3a06 100644 --- a/cinder/tests/unit/test_image_utils.py +++ b/cinder/tests/unit/test_image_utils.py @@ -1,5 +1,3 @@ -# Copyright (c) 2013 eNovance , Inc. -# All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -740,7 +738,8 @@ class TestUploadVolume(test.TestCase): ctxt = mock.sentinel.context image_service = mock.Mock() image_meta = {'id': 'test_id', - 'disk_format': input_format} + 'disk_format': input_format, + 'container_format': mock.sentinel.container_format} volume_path = mock.sentinel.volume_path mock_os.name = 'posix' data = mock_info.return_value @@ -778,7 +777,8 @@ class TestUploadVolume(test.TestCase): ctxt = mock.sentinel.context image_service = mock.Mock() image_meta = {'id': 'test_id', - 'disk_format': 'raw'} + 'disk_format': 'raw', + 'container_format': mock.sentinel.container_format} volume_path = mock.sentinel.volume_path mock_os.name = 'posix' mock_os.access.return_value = False @@ -796,6 +796,62 @@ class TestUploadVolume(test.TestCase): image_service.update.assert_called_once_with( ctxt, image_meta['id'], {}, mock_proxy.return_value) + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = True) + @mock.patch('eventlet.tpool.Proxy') + @mock.patch('cinder.image.image_utils.utils.temporary_chown') + @mock.patch('cinder.image.image_utils.CONF') + @mock.patch('six.moves.builtins.open') + @mock.patch('cinder.image.image_utils.qemu_img_info') + @mock.patch('cinder.image.image_utils.convert_image') + @mock.patch('cinder.image.image_utils.temporary_file') + @mock.patch('cinder.image.image_utils.os') + def test_same_format_compressed(self, mock_os, mock_temp, mock_convert, + mock_info, mock_open, mock_conf, + mock_chown, mock_proxy, + mock_engine_ready, mock_get_engine): + class fakeEngine(object): + + def __init__(self): + pass + + def compress_img(self, src, dest, run_as_root): + pass + + ctxt = mock.sentinel.context + image_service = mock.Mock() + image_meta = {'id': 'test_id', + 'disk_format': 'raw', + 'container_format': 'compressed'} + mock_conf.allow_compression_on_image_upload = True + volume_path = mock.sentinel.volume_path + mock_os.name = 'posix' + data = mock_info.return_value + data.file_format = 'raw' + data.backing_file = None + temp_file = mock_temp.return_value.__enter__.return_value + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + + output = image_utils.upload_volume(ctxt, image_service, image_meta, + volume_path) + + self.assertIsNone(output) + mock_convert.assert_called_once_with(volume_path, + temp_file, + 'raw', + compress=True, + run_as_root=True) + mock_info.assert_called_with(temp_file, run_as_root=True) + self.assertEqual(2, mock_info.call_count) + mock_open.assert_called_once_with(temp_file, 'rb') + mock_proxy.assert_called_once_with( + mock_open.return_value.__enter__.return_value) + image_service.update.assert_called_once_with( + ctxt, image_meta['id'], {}, mock_proxy.return_value) + mock_engine.compress_img.assert_called() + @mock.patch('eventlet.tpool.Proxy') @mock.patch('cinder.image.image_utils.utils.temporary_chown') @mock.patch('cinder.image.image_utils.CONF') @@ -810,7 +866,8 @@ class TestUploadVolume(test.TestCase): ctxt = mock.sentinel.context image_service = mock.Mock() image_meta = {'id': 'test_id', - 'disk_format': 'raw'} + 'disk_format': 'raw', + 'container_format': 'bare'} volume_path = mock.sentinel.volume_path mock_os.name = 'nt' mock_os.access.return_value = False @@ -827,6 +884,63 @@ class TestUploadVolume(test.TestCase): image_service.update.assert_called_once_with( ctxt, image_meta['id'], {}, mock_proxy.return_value) + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = True) + @mock.patch('eventlet.tpool.Proxy') + @mock.patch('cinder.image.image_utils.utils.temporary_chown') + @mock.patch('cinder.image.image_utils.CONF') + @mock.patch('six.moves.builtins.open') + @mock.patch('cinder.image.image_utils.qemu_img_info') + @mock.patch('cinder.image.image_utils.convert_image') + @mock.patch('cinder.image.image_utils.temporary_file') + @mock.patch('cinder.image.image_utils.os') + def test_same_format_on_nt_compressed(self, mock_os, mock_temp, + mock_convert, mock_info, + mock_open, mock_conf, + mock_chown, mock_proxy, + mock_engine_ready, mock_get_engine): + class fakeEngine(object): + + def __init__(self): + pass + + def compress_img(self, src, dest, run_as_root): + pass + + ctxt = mock.sentinel.context + image_service = mock.Mock() + image_meta = {'id': 'test_id', + 'disk_format': 'raw', + 'container_format': 'compressed'} + mock_conf.allow_compression_on_image_upload = True + volume_path = mock.sentinel.volume_path + mock_os.name = 'posix' + data = mock_info.return_value + data.file_format = 'raw' + data.backing_file = None + temp_file = mock_temp.return_value.__enter__.return_value + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + + output = image_utils.upload_volume(ctxt, image_service, image_meta, + volume_path) + + self.assertIsNone(output) + mock_convert.assert_called_once_with(volume_path, + temp_file, + 'raw', + compress=True, + run_as_root=True) + mock_info.assert_called_with(temp_file, run_as_root=True) + self.assertEqual(2, mock_info.call_count) + mock_open.assert_called_once_with(temp_file, 'rb') + mock_proxy.assert_called_once_with( + mock_open.return_value.__enter__.return_value) + image_service.update.assert_called_once_with( + ctxt, image_meta['id'], {}, mock_proxy.return_value) + mock_engine.compress_img.assert_called() + @mock.patch('cinder.image.image_utils.CONF') @mock.patch('six.moves.builtins.open') @mock.patch('cinder.image.image_utils.qemu_img_info') @@ -838,7 +952,8 @@ class TestUploadVolume(test.TestCase): ctxt = mock.sentinel.context image_service = mock.Mock() image_meta = {'id': 'test_id', - 'disk_format': mock.sentinel.disk_format} + 'disk_format': mock.sentinel.disk_format, + 'container_format': mock.sentinel.container_format} volume_path = mock.sentinel.volume_path mock_os.name = 'posix' data = mock_info.return_value @@ -1643,6 +1758,83 @@ class TestFetchToVolumeFormat(test.TestCase): image_utils.get_qemu_data, image_id, has_meta, disk_format_raw, dest, run_as_root=run_as_root) + @mock.patch('cinder.image.accelerator.is_gzip_compressed', + return_value = True) + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = True) + @mock.patch('cinder.image.image_utils.check_available_space') + @mock.patch('cinder.image.image_utils.convert_image') + @mock.patch('cinder.image.image_utils.volume_utils.copy_volume') + @mock.patch( + 'cinder.image.image_utils.replace_xenserver_image_with_coalesced_vhd') + @mock.patch('cinder.image.image_utils.is_xenserver_format', + return_value=False) + @mock.patch('cinder.image.image_utils.fetch') + @mock.patch('cinder.image.image_utils.qemu_img_info') + @mock.patch('cinder.image.image_utils.temporary_file') + @mock.patch('cinder.image.image_utils.CONF') + def test_defaults_compressed(self, mock_conf, mock_temp, mock_info, + mock_fetch, mock_is_xen, mock_repl_xen, + mock_copy, mock_convert, mock_check_space, + mock_engine_ready, mock_get_engine, + mock_gzip_compressed): + class fakeEngine(object): + def __init__(self): + pass + + def decompress_img(self, src, dest, run_as_root): + pass + + class FakeImageService(object): + def __init__(self, db_driver=None, + image_service=None, disk_format='raw'): + self.temp_images = None + self.disk_format = disk_format + + def show(self, context, image_id): + return {'size': 2 * units.Gi, + 'disk_format': self.disk_format, + 'container_format': 'compressed', + 'status': 'active'} + + ctxt = mock.sentinel.context + ctxt.user_id = mock.sentinel.user_id + image_service = FakeImageService() + image_id = mock.sentinel.image_id + dest = mock.sentinel.dest + volume_format = mock.sentinel.volume_format + out_subformat = None + blocksize = mock.sentinel.blocksize + + data = mock_info.return_value + data.file_format = volume_format + data.backing_file = None + data.virtual_size = 1234 + tmp = mock_temp.return_value.__enter__.return_value + + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + + output = image_utils.fetch_to_volume_format(ctxt, image_service, + image_id, dest, + volume_format, blocksize) + + self.assertIsNone(output) + mock_temp.assert_called_once_with() + mock_info.assert_has_calls([ + mock.call(tmp, force_share=False, run_as_root=True), + mock.call(tmp, run_as_root=True)]) + mock_fetch.assert_called_once_with(ctxt, image_service, image_id, + tmp, None, None) + self.assertFalse(mock_repl_xen.called) + self.assertFalse(mock_copy.called) + mock_convert.assert_called_once_with(tmp, dest, volume_format, + out_subformat=out_subformat, + run_as_root=True, + src_format='raw') + mock_engine.decompress_img.assert_called() + class TestXenserverUtils(test.TestCase): def test_is_xenserver_format(self): diff --git a/cinder/volume/api.py b/cinder/volume/api.py index 185590c0d41..0cef6a84359 100644 --- a/cinder/volume/api.py +++ b/cinder/volume/api.py @@ -1316,6 +1316,13 @@ class API(base.Base): pass recv_metadata = self.image_service.create(context, metadata) + + # NOTE(ZhengMa): Check if allow image compression before image + # uploading + if recv_metadata.get('container_format') == 'compressed': + allow_compression = CONF.allow_compression_on_image_upload + if allow_compression is False: + raise exception.ImageCompressionNotAllowed() except Exception: # NOTE(geguileo): To mimic behavior before conditional_update we # will rollback status if image create fails diff --git a/etc/cinder/rootwrap.d/volume.filters b/etc/cinder/rootwrap.d/volume.filters index 4df1d66fb11..0c51152f6a4 100644 --- a/etc/cinder/rootwrap.d/volume.filters +++ b/etc/cinder/rootwrap.d/volume.filters @@ -80,6 +80,8 @@ cgexec: ChainingRegExpFilter, cgexec, root, cgexec, -g, blkio:\S+ # cinder/image/image_utils.py qemu-img: EnvFilter, env, root, LC_ALL=C, qemu-img qemu-img_convert: CommandFilter, qemu-img, root +qzip: CommandFilter, qzip, root +gzip: CommandFilter, gzip, root # cinder/volume/nfs.py stat: CommandFilter, stat, root diff --git a/releasenotes/notes/leverage-compression-accelerator-579c7032290cd1e9.yaml b/releasenotes/notes/leverage-compression-accelerator-579c7032290cd1e9.yaml new file mode 100644 index 00000000000..f870aaf631b --- /dev/null +++ b/releasenotes/notes/leverage-compression-accelerator-579c7032290cd1e9.yaml @@ -0,0 +1,49 @@ +--- +features: + - | + A general framework to accommodate hardware compression accelerators for + compression of volumes uploaded to the Image service (Glance) as images + and decompression of compressed images used to create volumes is + introduced. + + The only accelerator supported in this release is Intel QuickAssist + Technology (QAT), which produces a compressed file in gzip format. + Refer to this `Cinder documentation + `_ + for more information about using this feature. + + Additionally, the framework provides software-based compression using + GUNzip tool if a suitable hardware accelerator is not available. + Because this software fallback could cause performance problems if the + Cinder services are not deployed on sufficiently powerful nodes, the + default setting is *not* to enable compression on image upload or + download. + + The compressed image of a volume will be stored in the + Image service (Glance) with the ``container_format`` image property of + ``compressed``. See the `Image service documentation + `_ for more information about + this image container format. +issues: + - | + In the Image service (Glance), the ``compressed`` container format + identifier does not indicate a particular compression technology; it is up + to the image consumer to determine what compression has been used, and + there is no requirement that OpenStack services must support arbitrary + compression technologies. For the upload and download of compressed + images, Cinder supports *only* the gzip format. + + While you may expect that Cinder will be able to consume any image in + ``compressed`` container format *that Cinder has created*, you should not + expect Cinder to be able to successfully use an image in ``compressed`` + format that it has not created itself. +upgrade: + - | + Added string config option ``compression_format`` in [default] section of + cinder.conf to specify image compression format. Currently the only legal + value for this option is ``gzip``. + - | + Added boolean config option ``allow_compression_on_image_upload`` in + [default] section of cinder.conf to enable/disable image compression on + image upload. The default value of this option is ``false``, which means + image compression is disabled.