From e0c54408195ba3a17a0d0e6ad7e5fc7bb789895d Mon Sep 17 00:00:00 2001 From: Erno Kuvaja Date: Wed, 18 Mar 2020 18:31:35 +0000 Subject: [PATCH] Add decompression import plugin Supported compression formats initially are: * zip * gzip * lha/lzh _if_ lhafile is installed Change-Id: Id125ebb5e8a9b22a8797d3158e60451d80bfaa14 --- .../admin/interoperable-image-import.rst | 74 ++++++++ .../flows/plugins/image_decompression.py | 166 ++++++++++++++++++ ...decompression_plugin-5f085666aae01f29.yaml | 7 + setup.cfg | 1 + 4 files changed, 248 insertions(+) create mode 100644 glance/async_/flows/plugins/image_decompression.py create mode 100644 releasenotes/notes/image_decompression_plugin-5f085666aae01f29.yaml diff --git a/doc/source/admin/interoperable-image-import.rst b/doc/source/admin/interoperable-image-import.rst index 66ec0c779c..2a896b40f5 100644 --- a/doc/source/admin/interoperable-image-import.rst +++ b/doc/source/admin/interoperable-image-import.rst @@ -592,6 +592,80 @@ You will need to configure 'glance-image-import.conf' file as shown below: [image_conversion] output_format = raw +The Image Decompression +----------------------- +.. list-table:: + + * - release introduced + - Ussuri (Glance 20.0.0) + * - configuration file + - ``glance-image-import.conf`` + +This plugin implements automated image decompression for Interoperable Image +Import. One use case for this plugin would be environments where user or +operator wants to use 'web-download' method and the image provider supplies +only compressed images. + +.. note:: + + This plugin may only be used as part of the interoperable image import + workflow (``POST v2/images/{image_id}/import``). *It has no effect on the + image data upload call* (``PUT v2/images/{image_id}/file``). + + You can guarantee that your end users must use interoperable image import by + restricting the ``upload_image`` policy appropriately in the Glance + ``policy.json`` file. By default, this policy is unrestricted (that is, + any authorized user may make the image upload call). + + For example, to allow only admin or service users to make the image upload + call, the policy could be restricted as follows: + + .. code-block:: text + + "upload_image": "role:admin or (service_user_id:) or + (service_roles:)" + + where "service_role" is the role which is created for the service user + and assigned to trusted services. + +To use the Image Decompression Plugin, the following configuration is +required. + +You will need to add "image_decompression" to 'glance-image-import.conf' file +as shown below: + +.. code-block:: ini + + [image_import_opts] + image_import_plugins = ['image_decompression'] + +.. note:: + + The supported archive types for Image Decompression are zip, lha/lzh and gzip. + Currently the plugin does not support multi-layered archives (like tar.gz). + Lha/lzh is only supported in case python3 `lhafile` dependency library is + installed, absence of this dependency will fail the import job where lha file + is provided. (In this case we know it won't be bootable as the image is + compressed and we do not have means to decompress it.) + +.. note:: + + ``image_import_plugins`` config option is a list and multiple plugins can be + enabled for the import flow. The plugins are not run in parallel. One can + enable multiple plugins by configuring them in the + ``glance-image-import.conf`` for example as following: + + .. code-block:: ini + + [image_import_opts] + image_import_plugins = ['image_decompression', 'image_conversion'] + + [image_conversion] + output_format = raw + + If Image Conversion is used together, decompression must happen first, this + is ensured by ordering the plugins. + .. _glance-api.conf: https://opendev.org/openstack/glance/src/branch/master/etc/glance-api.conf .. _glance-image-import.conf.sample: https://opendev.org/openstack/glance/src/branch/master/etc/glance-image-import.conf.sample .. _`Image Import Refactor`: https://specs.openstack.org/openstack/glance-specs/specs/mitaka/approved/image-import/image-import-refactor.html diff --git a/glance/async_/flows/plugins/image_decompression.py b/glance/async_/flows/plugins/image_decompression.py new file mode 100644 index 0000000000..282e6be3c4 --- /dev/null +++ b/glance/async_/flows/plugins/image_decompression.py @@ -0,0 +1,166 @@ +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import gzip +import os +import shutil +import zipfile + +from oslo_log import log as logging +from oslo_utils import encodeutils +from taskflow.patterns import linear_flow as lf +from taskflow import task + +LOG = logging.getLogger(__name__) + +# Note(jokke): The number before '_' is offset for the magic number in header +MAGIC_NUMBERS = { + '0_zipfile': bytes([0x50, 0x4B, 0x03, 0x04]), + '2_lhafile': bytes([0x2D, 0x6C, 0x68]), + '0_gzipfile': bytes([0x1F, 0x8B, 0x08])} + +NO_LHA = False + +try: + import lhafile +except ImportError: + LOG.debug("No lhafile available.") + NO_LHA = True + + +def header_lengths(): + headers = [] + for key, val in MAGIC_NUMBERS.items(): + offset, key = key.split("_") + headers.append(int(offset) + len(val)) + return headers + + +MAX_HEADER = max(header_lengths()) + + +def _zipfile(src_path, dest_path, image_id): + try: + with zipfile.ZipFile(src_path, 'r') as zfd: + content = zfd.namelist() + if len(content) != 1: + raise Exception("Archive contains more than one file.") + else: + zfd.extract(content[0], dest_path) + except Exception as e: + LOG.debug("ZIP: Error decompressing image %(iid)s: %(msg)s", { + "iid": image_id, + "msg": encodeutils.exception_to_unicode(e)}) + raise + + +def _lhafile(src_path, dest_path, image_id): + if NO_LHA: + raise Exception("No lhafile available.") + try: + with lhafile.LhaFile(src_path, 'r') as lfd: + content = lfd.namelist() + if len(content) != 1: + raise Exception("Archive contains more than one file.") + else: + lfd.extract(content[0], dest_path) + except Exception as e: + LOG.debug("LHA: Error decompressing image %(iid)s: %(msg)s", { + "iid": image_id, + "msg": encodeutils.exception_to_unicode(e)}) + raise + + +def _gzipfile(src_path, dest_path, image_id): + try: + with gzip.open(src_path, 'r') as gzfd: + with open(dest_path, 'wb') as fd: + shutil.copyfileobj(gzfd, fd) + except gzip.BadGzipFile as e: + LOG.debug("ZIP: Error decompressing image %(iid)s: Bad GZip file: " + "%(msg)s", {"iid": image_id, + "msg": encodeutils.exception_to_unicode(e)}) + raise + except Exception as e: + LOG.debug("GZIP: Error decompressing image %(iid)s: %(msg)s", { + "iid": image_id, + "msg": encodeutils.exception_to_unicode(e)}) + raise + + +class _DecompressImage(task.Task): + + default_provides = 'file_path' + + def __init__(self, context, task_id, task_type, + image_repo, image_id): + self.context = context + self.task_id = task_id + self.task_type = task_type + self.image_repo = image_repo + self.image_id = image_id + self.dest_path = "" + super(_DecompressImage, self).__init__( + name='%s-Decompress_Image-%s' % (task_type, task_id)) + + def execute(self, file_path, **kwargs): + + # TODO(jokke): Once we support other schemas we need to take them into + # account and handle the paths here. + src_path = file_path.split('file://')[-1] + self.dest_path = "%(path)s.uc" % {'path': src_path} + head = None + with open(src_path, 'rb') as fd: + head = fd.read(MAX_HEADER) + for key, val in MAGIC_NUMBERS.items(): + offset, key = key.split("_") + offset = int(offset) + key = "_" + key + if head.startswith(val, offset): + globals()[key](src_path, self.dest_path, self.image_id) + os.replace(self.dest_path, src_path) + + return "file://%s" % src_path + + def revert(self, result=None, **kwargs): + # NOTE(flaper87, jokke): If result is None, it probably + # means this task failed. Otherwise, we would have + # a result from its execution. This includes the case + # that nothing was to be compressed. + if result is not None: + LOG.debug("Image decompression failed.") + if os.path.exists(self.dest_path): + os.remove(self.dest_path) + + +def get_flow(**kwargs): + """Return task flow for no-op. + + :param context: request context + :param task_id: Task ID. + :param task_type: Type of the task. + :param image_repo: Image repository used. + :param image_id: Image ID + """ + context = kwargs.get('context') + task_id = kwargs.get('task_id') + task_type = kwargs.get('task_type') + image_repo = kwargs.get('image_repo') + image_id = kwargs.get('image_id') + + return lf.Flow(task_type).add( + _DecompressImage(context, task_id, task_type, + image_repo, image_id), + ) diff --git a/releasenotes/notes/image_decompression_plugin-5f085666aae01f29.yaml b/releasenotes/notes/image_decompression_plugin-5f085666aae01f29.yaml new file mode 100644 index 0000000000..1986c3588c --- /dev/null +++ b/releasenotes/notes/image_decompression_plugin-5f085666aae01f29.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + New Interoperable Image Import plugin has been introduced + to address the use case of providing compressed images + either through 'web-download' or to optimize the network + utilization between the client and Glance. diff --git a/setup.cfg b/setup.cfg index cbbc6bbbe2..d71a6aa722 100644 --- a/setup.cfg +++ b/setup.cfg @@ -81,6 +81,7 @@ glance.image_import.plugins = no_op = glance.async_.flows.plugins.no_op:get_flow inject_image_metadata=glance.async_.flows.plugins.inject_image_metadata:get_flow image_conversion=glance.async_.flows.plugins.image_conversion:get_flow + image_decompression=glance.async_.flows.plugins.image_decompression:get_flow glance.image_import.internal_plugins = web_download = glance.async_.flows._internal_plugins.web_download:get_flow