From c6a663ea6da98243c54f1e99547e76aeab945699 Mon Sep 17 00:00:00 2001 From: Erno Kuvaja Date: Tue, 5 Jun 2018 19:21:08 +0100 Subject: [PATCH] Add image conversion plugin This change adds image conversion plugin for Interoperable Image Import workflow. Change-Id: Icd4f4f30b12f6dd073dcdbf7f842e933c75ae249 --- .../admin/interoperable-image-import.rst | 83 ++++++++- etc/glance-image-import.conf.sample | 28 +++ .../async/flows/plugins/image_conversion.py | 165 ++++++++++++++++++ glance/async/flows/plugins/plugin_opts.py | 3 + glance/domain/__init__.py | 6 +- ...ge-conversion-plugin-5aee45e1a1a5bb2b.yaml | 14 ++ setup.cfg | 1 + 7 files changed, 297 insertions(+), 3 deletions(-) create mode 100644 glance/async/flows/plugins/image_conversion.py create mode 100644 releasenotes/notes/image-conversion-plugin-5aee45e1a1a5bb2b.yaml diff --git a/doc/source/admin/interoperable-image-import.rst b/doc/source/admin/interoperable-image-import.rst index dd1199c45a..f93fdc0c74 100644 --- a/doc/source/admin/interoperable-image-import.rst +++ b/doc/source/admin/interoperable-image-import.rst @@ -377,6 +377,87 @@ required. See the :ref:`property-protections` section of this Guide for more information. +The Image Conversion +-------------------- +.. list-table:: + + * - release introduced + - Rocky (Glance 17.0.0) + * - configuration file + - ``glance-image-import.conf`` + * - configuration file section + - ``[image_conversion]`` + +This plugin implements automated image conversion for Interoperable Image +Import. One use case for this plugin would be environments where Ceph is used +as image back-end and operators want to optimize the back-end capabilities by +ensuring that all images will be in raw format while not putting the burden of +converting the images to their end users. + +.. note:: + + This plugin may only be used as part of the interoperable image import + workflow (``POST v2/images/{image_id}/import``). *It has no effect on the + image data upload call* (``PUT v2/images/{image_id}/file``). + + You can guarantee that your end users must use interoperable image import by + restricting the ``upload_image`` policy appropriately in the Glance + ``policy.json`` file. By default, this policy is unrestricted (that is, + any authorized user may make the image upload call). + + For example, to allow only admin or service users to make the image upload + call, the policy could be restricted as follows: + + .. code-block:: text + + "upload_image": "role:admin or (service_user_id:) or + (service_roles:)" + + where "service_role" is the role which is created for the service user + and assigned to trusted services. + +To use the Image Conversion Plugin, the following configuration is +required. + +You will need to configure 'glance-image-import.conf' file as shown below: + + .. code-block:: ini + + [image_import_opts] + image_import_plugins = ['image_conversion'] + + [image_conversion] + output_format = raw + +.. note:: + + The default output format is raw in which case there is no need to have + 'image_conversion' section and its 'output_format' defined in the config + file. + + The input format needs to be one of the `qemu-img supported ones`_ for this + feature to work. In case of qemu-img call failing on the source image the + import process will fail if 'image_conversion' plugin is enabled. + +.. note:: + + ``image_import_plugins`` config option is a list and multiple plugins can be + enabled for the import flow. The plugins are not run in parallel. One can + enable multiple plugins by configuring them in the + ``glance-image-import.conf`` for example as following: + + .. code-block:: ini + + [image_import_opts] + image_import_plugins = ['inject_image_metadata', 'image_conversion'] + + [inject_metadata_properties] + ignore_user_roles = admin,... + inject = "property1":"value1","property2":"value2",... + + [image_conversion] + output_format = raw + .. _glance-api.conf: http://git.openstack.org/cgit/openstack/glance/tree/etc/glance-api.conf .. _glance-image-import.conf.sample: http://git.openstack.org/cgit/openstack/glance/tree/etc/glance-image-import.conf.sample .. _`Image Import Refactor`: https://specs.openstack.org/openstack/glance-specs/specs/mitaka/approved/image-import/image-import-refactor.html @@ -387,4 +468,4 @@ required. .. _`Stevedore`: https://docs.openstack.org/stevedore .. _`Taskflow`: https://docs.openstack.org/taskflow .. _`Taskflow "Task" object`: https://docs.openstack.org/taskflow/latest/user/atoms.html#task - +.. _`qemu-img supported ones`: https://github.com/qemu/qemu/blob/master/qemu-img.texi#L599-L725 diff --git a/etc/glance-image-import.conf.sample b/etc/glance-image-import.conf.sample index f1b7f63554..4277692504 100644 --- a/etc/glance-image-import.conf.sample +++ b/etc/glance-image-import.conf.sample @@ -1,6 +1,34 @@ [DEFAULT] +[image_conversion] + +# +# From glance +# + +# +# Desired output format for image conversion plugin. +# +# Provide a valid image format to which the conversion plugin +# will convert the image before storing it to the back-end. +# +# Note, if the Image Conversion plugin for image import is defined, users +# should only upload disk formats that are supported by `quemu-img` otherwise +# the conversion and import will fail. +# +# Possible values: +# * qcow2 +# * raw +# * vdmk +# +# Related Options: +# * disk_formats +# (string value) +# Allowed values: qcow2, raw, vdmk +#output_format = raw + + [image_import_opts] # diff --git a/glance/async/flows/plugins/image_conversion.py b/glance/async/flows/plugins/image_conversion.py new file mode 100644 index 0000000000..1d7adcf53d --- /dev/null +++ b/glance/async/flows/plugins/image_conversion.py @@ -0,0 +1,165 @@ +# Copyright 2018 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import os + +from oslo_concurrency import processutils as putils +from oslo_config import cfg +from oslo_log import log as logging +from oslo_utils import encodeutils +from oslo_utils import excutils +from taskflow.patterns import linear_flow as lf +from taskflow import task + +from glance.async import utils +from glance.i18n import _ + +LOG = logging.getLogger(__name__) + +conversion_plugin_opts = [ + cfg.StrOpt('output_format', + default='raw', + choices=('qcow2', 'raw', 'vdmk'), + help=_(""" +Desired output format for image conversion plugin. + +Provide a valid image format to which the conversion plugin +will convert the image before storing it to the back-end. + +Note, if the Image Conversion plugin for image import is defined, users +should only upload disk formats that are supported by `quemu-img` otherwise +the conversion and import will fail. + +Possible values: + * qcow2 + * raw + * vdmk + +Related Options: + * disk_formats +""")), +] + +CONF = cfg.CONF + +CONF.register_opts(conversion_plugin_opts, group='image_conversion') + + +class _ConvertImage(task.Task): + + default_provides = 'file_path' + + def __init__(self, context, task_id, task_type, + image_repo, image_id): + self.context = context + self.task_id = task_id + self.task_type = task_type + self.image_repo = image_repo + self.image_id = image_id + self.dest_path = "" + super(_ConvertImage, self).__init__( + name='%s-Convert_Image-%s' % (task_type, task_id)) + + def execute(self, file_path, **kwargs): + + target_format = CONF.conversion_plugin_options.output_format + # TODO(jokke): Once we support other schemas we need to take them into + # account and handle the paths here. + src_path = file_path.split('file://')[-1] + dest_path = "%(path)s.%(target)s" % {'path': src_path, + 'target': target_format} + self.dest_path = dest_path + + try: + stdout, stderr = putils.trycmd("qemu-img", "info", + "--output=json", + src_path, + prlimit=utils.QEMU_IMG_PROC_LIMITS, + log_errors=putils.LOG_ALL_ERRORS,) + except OSError as exc: + with excutils.save_and_reraise_exception(): + exc_message = encodeutils.exception_to_unicode(exc) + msg = ("Failed to do introspection as part of image " + "conversion for %(iid)s: %(err)s") + LOG.error(msg, {'iid': self.image_id, 'err': exc_message}) + + if stderr: + raise RuntimeError(stderr) + + metadata = json.loads(stdout) + source_format = metadata.get('format') + virtual_size = metadata.get('virtual-size', 0) + image = self.image_repo.get(self.image_id) + image.virtual_size = virtual_size + + if source_format == target_format: + LOG.debug("Source is already in target format, " + "not doing conversion for %s", self.image_id) + self.image_repo.save(image) + return file_path + + try: + stdout, stderr = putils.trycmd('qemu-img', 'convert', + '-f', source_format, + '-O', target_format, + src_path, dest_path, + log_errors=putils.LOG_ALL_ERRORS) + except OSError as exc: + with excutils.save_and_reraise_exception(): + exc_message = encodeutils.exception_to_unicode(exc) + msg = "Failed to do image conversion for %(iid)s: %(err)s" + LOG.error(msg, {'iid': self.image_id, 'err': exc_message}) + + if stderr: + raise RuntimeError(stderr) + + image.disk_format = target_format + image.container_format = 'bare' + self.image_repo.save(image) + + os.remove(src_path) + + return "file://%s" % dest_path + + def revert(self, result=None, **kwargs): + # NOTE(flaper87): If result is None, it probably + # means this task failed. Otherwise, we would have + # a result from its execution. + if result is not None: + LOG.debug("Image conversion failed.") + if os.path.exists(self.dest_path): + os.remove(self.dest_path) + + +def get_flow(**kwargs): + """Return task flow for no-op. + + :param context: request context + :param task_id: Task ID. + :param task_type: Type of the task. + :param image_repo: Image repository used. + :param image_id: Image ID + """ + context = kwargs.get('context') + task_id = kwargs.get('task_id') + task_type = kwargs.get('task_type') + image_repo = kwargs.get('image_repo') + image_id = kwargs.get('image_id') + + return lf.Flow(task_type).add( + _ConvertImage(context, task_id, task_type, + image_repo, image_id), + ) diff --git a/glance/async/flows/plugins/plugin_opts.py b/glance/async/flows/plugins/plugin_opts.py index 440e1eb7e3..709b78ce4e 100644 --- a/glance/async/flows/plugins/plugin_opts.py +++ b/glance/async/flows/plugins/plugin_opts.py @@ -14,6 +14,7 @@ # under the License. +import glance.async.flows.plugins.image_conversion import glance.async.flows.plugins.inject_image_metadata @@ -27,6 +28,8 @@ import glance.async.flows.plugins.inject_image_metadata PLUGIN_OPTS = [ ('inject_metadata_properties', glance.async.flows.plugins.inject_image_metadata.inject_metadata_opts), + ('image_conversion', + glance.async.flows.plugins.image_conversion.conversion_plugin_opts), ] diff --git a/glance/domain/__init__.py b/glance/domain/__init__.py index acf46ea637..7cd6aadf53 100644 --- a/glance/domain/__init__.py +++ b/glance/domain/__init__.py @@ -194,7 +194,8 @@ class Image(object): @container_format.setter def container_format(self, value): - if hasattr(self, '_container_format') and self.status != 'queued': + if (hasattr(self, '_container_format') and + self.status not in ('queued', 'importing')): msg = _("Attribute container_format can be only replaced " "for a queued image.") raise exception.Forbidden(message=msg) @@ -206,7 +207,8 @@ class Image(object): @disk_format.setter def disk_format(self, value): - if hasattr(self, '_disk_format') and self.status != 'queued': + if (hasattr(self, '_disk_format') and + self.status not in ('queued', 'importing')): msg = _("Attribute disk_format can be only replaced " "for a queued image.") raise exception.Forbidden(message=msg) diff --git a/releasenotes/notes/image-conversion-plugin-5aee45e1a1a5bb2b.yaml b/releasenotes/notes/image-conversion-plugin-5aee45e1a1a5bb2b.yaml new file mode 100644 index 0000000000..d4ffcdcfe2 --- /dev/null +++ b/releasenotes/notes/image-conversion-plugin-5aee45e1a1a5bb2b.yaml @@ -0,0 +1,14 @@ +--- +prelude: > + Automatic image conversion plugin for Interoperable Image Import. This + release introduces a new plugin that can be used to convert images to + specific format automatically upon image import. +features: + - | + Automatic image conversion plugin for Interoperable Image Import. With + this release operators can specify target image format and get all images + created via the Image Import methods introduced in the Images API v2.6 + converted automatically to that format. The feautre uses qemu-img under + the hood which limits the source image formats that users can upload. Any + image that fails the conversion when this plugin is enabled will fail the + image creation. diff --git a/setup.cfg b/setup.cfg index ac08e1bcee..9e373f5371 100644 --- a/setup.cfg +++ b/setup.cfg @@ -75,6 +75,7 @@ glance.flows.import = glance.image_import.plugins = no_op = glance.async.flows.plugins.no_op:get_flow inject_image_metadata=glance.async.flows.plugins.inject_image_metadata:get_flow + image_conversion=glance.async.flows.plugins.image_conversion:get_flow glance.image_import.internal_plugins = web_download = glance.async.flows._internal_plugins.web_download:get_flow