From 480ea3825fde9d2a317064416a1304a1d3c0cf53 Mon Sep 17 00:00:00 2001 From: Pierre-Samuel Le Stang Date: Tue, 3 May 2022 16:47:54 +0200 Subject: [PATCH] Implement glance-download internal plugin Add a new import method called glance-download that implements a glance to glance download in a multi-region cloud with a federated Keystone. This method will copy the image data and selected metadata to the target glance, checking that the downloaded size match the "size" image attribute in the source glance. Implements: blueprint glance-download-import Co-Authored-By: Victor Coutellier Change-Id: Ic51c5fd87caf04d38aeaf758ad2d0e2f28098e4d --- glance/api/v2/images.py | 23 +- .../flows/_internal_plugins/base_download.py | 131 +++++++++++ .../_internal_plugins/glance_download.py | 115 ++++++++++ .../flows/_internal_plugins/web_download.py | 106 +-------- glance/async_/flows/api_image_import.py | 151 ++++++++++++- glance/common/config.py | 2 +- .../async_/flows/test_api_image_import.py | 159 +++++++++++++- .../unit/async_/flows/test_base_download.py | 207 ++++++++++++++++++ .../unit/async_/flows/test_glance_download.py | 167 ++++++++++++++ .../unit/async_/flows/test_web_download.py | 196 +++-------------- glance/tests/unit/utils.py | 6 + .../unit/v2/test_discovery_image_import.py | 2 +- glance/tests/unit/v2/test_images_resource.py | 41 +++- ...ance-download-method-be6d9e927b8b0a43.yaml | 10 + setup.cfg | 1 + 15 files changed, 1039 insertions(+), 278 deletions(-) create mode 100644 glance/async_/flows/_internal_plugins/base_download.py create mode 100644 glance/async_/flows/_internal_plugins/glance_download.py create mode 100644 glance/tests/unit/async_/flows/test_base_download.py create mode 100644 glance/tests/unit/async_/flows/test_glance_download.py create mode 100644 releasenotes/notes/add-glance-download-method-be6d9e927b8b0a43.yaml diff --git a/glance/api/v2/images.py b/glance/api/v2/images.py index 282069cec0..95cfe7b9c3 100644 --- a/glance/api/v2/images.py +++ b/glance/api/v2/images.py @@ -18,6 +18,7 @@ import http.client as http import os import re import urllib.parse as urlparse +import uuid from castellan.common import exception as castellan_exception from castellan import key_manager @@ -332,9 +333,10 @@ class ImagesController(object): msg = _("Only images with status active can be targeted for " "copying") raise exception.Conflict(msg) - if image.status != 'queued' and import_method == 'web-download': + if (image.status != 'queued' and + import_method in ['web-download', 'glance-download']): msg = _("Image needs to be in 'queued' state to use " - "'web-download' method") + "'%s' method") % import_method raise exception.Conflict(msg) if (image.status != 'uploading' and import_method == 'glance-direct'): @@ -347,6 +349,23 @@ class ImagesController(object): if not getattr(image, 'disk_format', None): msg = _("'disk_format' needs to be set before import") raise exception.Conflict(msg) + if import_method == 'glance-download': + if 'glance_region' not in body.get('method'): + msg = _("'glance_region' needs to be set for " + "glance-download import method") + raise webob.exc.HTTPBadRequest(explanation=msg) + if 'glance_image_id' not in body.get('method'): + msg = _("'glance_image_id' needs to be set for " + "glance-download import method") + raise webob.exc.HTTPBadRequest(explanation=msg) + try: + uuid.UUID(body['method']['glance_image_id']) + except ValueError: + msg = (_("Remote image id does not look like a UUID: %s") + % body['method']['glance_image_id']) + raise webob.exc.HTTPBadRequest(explanation=msg) + if 'glance_service_interface' not in body.get('method'): + body.get('method')['glance_service_interface'] = 'public' # NOTE(danms): For copy-image only, we check policy to decide # if the user should be able to do this. Otherwise, we forbid diff --git a/glance/async_/flows/_internal_plugins/base_download.py b/glance/async_/flows/_internal_plugins/base_download.py new file mode 100644 index 0000000000..a8f90b9263 --- /dev/null +++ b/glance/async_/flows/_internal_plugins/base_download.py @@ -0,0 +1,131 @@ +# Copyright 2018 Red Hat, Inc. +# Copyright 2022 OVHCloud +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc + +import glance_store as store_api +from glance_store import backend +from oslo_config import cfg +from oslo_log import log as logging +import six +from taskflow import task + +from glance.common import exception +from glance.i18n import _, _LE + +LOG = logging.getLogger(__name__) + +CONF = cfg.CONF + + +@six.add_metaclass(abc.ABCMeta) +class BaseDownload(task.Task): + + default_provides = 'file_uri' + + def __init__(self, task_id, task_type, action_wrapper, stores, + plugin_name): + self.task_id = task_id + self.task_type = task_type + self.image_id = action_wrapper.image_id + self.action_wrapper = action_wrapper + self.stores = stores + self._path = None + self.plugin_name = plugin_name or 'Download' + super(BaseDownload, self).__init__( + name='%s-%s-%s' % (task_type, self.plugin_name, task_id)) + + # NOTE(abhishekk): Use reserved 'os_glance_staging_store' for + # staging the data, the else part will be removed once old way + # of configuring store is deprecated. + if CONF.enabled_backends: + self.store = store_api.get_store_from_store_identifier( + 'os_glance_staging_store') + else: + if CONF.node_staging_uri is None: + msg = (_("%(task_id)s of %(task_type)s not configured " + "properly. Missing node_staging_uri: %(work_dir)s") % + {'task_id': self.task_id, + 'task_type': self.task_type, + 'work_dir': CONF.node_staging_uri}) + raise exception.BadTaskConfiguration(msg) + + self.store = self._build_store() + + def _build_store(self): + # NOTE(flaper87): Due to the nice glance_store api (#sarcasm), we're + # forced to build our own config object, register the required options + # (and by required I mean *ALL* of them, even the ones we don't want), + # and create our own store instance by calling a private function. + # This is certainly unfortunate but it's the best we can do until the + # glance_store refactor is done. A good thing is that glance_store is + # under our team's management and it gates on Glance so changes to + # this API will (should?) break task's tests. + # TODO(abhishekk): After removal of backend module from glance_store + # need to change this to use multi_backend module. + conf = cfg.ConfigOpts() + try: + backend.register_opts(conf) + except cfg.DuplicateOptError: + pass + + conf.set_override('filesystem_store_datadir', + CONF.node_staging_uri[7:], + group='glance_store') + + # NOTE(flaper87): Do not even try to judge me for this... :( + # With the glance_store refactor, this code will change, until + # that happens, we don't have a better option and this is the + # least worst one, IMHO. + store = store_api.backend._load_store(conf, 'file') + + if store is None: + msg = (_("%(task_id)s of %(task_type)s not configured " + "properly. Could not load the filesystem store") % + {'task_id': self.task_id, 'task_type': self.task_type}) + raise exception.BadTaskConfiguration(msg) + + store.configure() + return store + + def revert(self, result, **kwargs): + LOG.error(_LE('Task: %(task_id)s failed to import image ' + '%(image_id)s to the filesystem.'), + {'task_id': self.task_id, + 'image_id': self.image_id}) + # NOTE(abhishekk): Revert image state back to 'queued' as + # something went wrong. + # NOTE(danms): If we failed to stage the image, then none + # of the _ImportToStore() tasks could have run, so we need + # to move all stores out of "importing" and into "failed". + with self.action_wrapper as action: + action.set_image_attribute(status='queued') + action.remove_importing_stores(self.stores) + action.add_failed_stores(self.stores) + + # NOTE(abhishekk): Deleting partial image data from staging area + if self._path is not None: + LOG.debug(('Deleting image %(image_id)s from staging ' + 'area.'), {'image_id': self.image_id}) + try: + if CONF.enabled_backends: + store_api.delete(self._path, None) + else: + store_api.delete_from_backend(self._path) + except Exception: + LOG.exception(_LE("Error reverting web/glance download " + "task: %(task_id)s"), { + 'task_id': self.task_id}) diff --git a/glance/async_/flows/_internal_plugins/glance_download.py b/glance/async_/flows/_internal_plugins/glance_download.py new file mode 100644 index 0000000000..3eefe31455 --- /dev/null +++ b/glance/async_/flows/_internal_plugins/glance_download.py @@ -0,0 +1,115 @@ +# Copyright 2022 OVHCloud +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import urllib.request + +from oslo_config import cfg +from oslo_log import log as logging +from oslo_utils import encodeutils +from oslo_utils import excutils +from taskflow.patterns import linear_flow as lf + +from glance.async_.flows._internal_plugins import base_download +from glance.async_ import utils +from glance.common import exception +from glance.common import utils as common_utils +from glance.i18n import _, _LI, _LE + +LOG = logging.getLogger(__name__) + +CONF = cfg.CONF + + +class _DownloadGlanceImage(base_download.BaseDownload): + + def __init__(self, context, task_id, task_type, action_wrapper, stores, + glance_region, glance_image_id, glance_service_interface): + self.context = context + self.glance_region = glance_region + self.glance_image_id = glance_image_id + self.glance_service_interface = glance_service_interface + super(_DownloadGlanceImage, + self).__init__(task_id, task_type, action_wrapper, stores, + 'GlanceDownload') + + def execute(self, image_size): + """Create temp file into store and return path to it + + :param image_size: Glance Image Size retrieved from ImportMetadata task + """ + try: + glance_endpoint = utils.get_glance_endpoint( + self.context, + self.glance_region, + self.glance_service_interface) + image_download_url = '%s/v2/images/%s/file' % ( + glance_endpoint, self.glance_image_id) + if not common_utils.validate_import_uri(image_download_url): + LOG.debug("Processed URI for glance-download does not pass " + "filtering: %s", image_download_url) + msg = (_("Processed URI for glance-download does not pass " + "filtering: %s") % image_download_url) + raise exception.ImportTaskError(msg) + LOG.info(_LI("Downloading glance image %s"), image_download_url) + token = self.context.auth_token + request = urllib.request.Request(image_download_url, + headers={'X-Auth-Token': token}) + data = urllib.request.urlopen(request) + except Exception as e: + with excutils.save_and_reraise_exception(): + LOG.error( + _LE("Task %(task_id)s failed with exception %(error)s"), { + "error": encodeutils.exception_to_unicode(e), + "task_id": self.task_id + }) + + self._path, bytes_written = self.store.add(self.image_id, data, 0)[0:2] + if bytes_written != image_size: + msg = (_("Task %(task_id)s failed because downloaded data " + "size %(data_size)i is different from expected %(" + "expected)i") % + {"task_id": self.task_id, "data_size": bytes_written, + "expected": image_size}) + raise exception.ImportTaskError(msg) + return self._path + + +def get_flow(**kwargs): + """Return task flow for no-op. + + :param context: request context + :param task_id: Task ID. + :param task_type: Type of the task. + :param image_repo: Image repository used. + :param image_id: Image ID + :param source_region: Source region name + """ + context = kwargs.get('context') + task_id = kwargs.get('task_id') + task_type = kwargs.get('task_type') + action_wrapper = kwargs.get('action_wrapper') + stores = kwargs.get('backend', [None]) + # glance-download parameters + import_req = kwargs.get('import_req') + method = import_req.get('method') + glance_region = method.get('glance_region') + glance_image_id = method.get('glance_image_id') + glance_service_interface = method.get('glance_service_interface') + + return lf.Flow(task_type).add( + _DownloadGlanceImage(context, task_id, task_type, action_wrapper, + stores, glance_region, glance_image_id, + glance_service_interface), + ) diff --git a/glance/async_/flows/_internal_plugins/web_download.py b/glance/async_/flows/_internal_plugins/web_download.py index 98e497c3c3..e14e4431f0 100644 --- a/glance/async_/flows/_internal_plugins/web_download.py +++ b/glance/async_/flows/_internal_plugins/web_download.py @@ -1,4 +1,5 @@ # Copyright 2018 Red Hat, Inc. +# Copyright 2022 OVHCloud # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -12,92 +13,29 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import glance_store as store_api -from glance_store import backend + from oslo_config import cfg from oslo_log import log as logging from oslo_utils import encodeutils from oslo_utils import excutils from taskflow.patterns import linear_flow as lf -from taskflow import task -from taskflow.types import failure +from glance.async_.flows._internal_plugins import base_download from glance.common import exception from glance.common.scripts import utils as script_utils -from glance.i18n import _, _LE +from glance.i18n import _ LOG = logging.getLogger(__name__) CONF = cfg.CONF -class _WebDownload(task.Task): - - default_provides = 'file_uri' +class _WebDownload(base_download.BaseDownload): def __init__(self, task_id, task_type, uri, action_wrapper, stores): - self.task_id = task_id - self.task_type = task_type - self.image_id = action_wrapper.image_id self.uri = uri - self.action_wrapper = action_wrapper - self.stores = stores - self._path = None - super(_WebDownload, self).__init__( - name='%s-WebDownload-%s' % (task_type, task_id)) - - # NOTE(abhishekk): Use reserved 'os_glance_staging_store' for - # staging the data, the else part will be removed once old way - # of configuring store is deprecated. - if CONF.enabled_backends: - self.store = store_api.get_store_from_store_identifier( - 'os_glance_staging_store') - else: - if CONF.node_staging_uri is None: - msg = (_("%(task_id)s of %(task_type)s not configured " - "properly. Missing node_staging_uri: %(work_dir)s") % - {'task_id': self.task_id, - 'task_type': self.task_type, - 'work_dir': CONF.node_staging_uri}) - raise exception.BadTaskConfiguration(msg) - - self.store = self._build_store() - - def _build_store(self): - # NOTE(flaper87): Due to the nice glance_store api (#sarcasm), we're - # forced to build our own config object, register the required options - # (and by required I mean *ALL* of them, even the ones we don't want), - # and create our own store instance by calling a private function. - # This is certainly unfortunate but it's the best we can do until the - # glance_store refactor is done. A good thing is that glance_store is - # under our team's management and it gates on Glance so changes to - # this API will (should?) break task's tests. - # TODO(abhishekk): After removal of backend module from glance_store - # need to change this to use multi_backend module. - conf = cfg.ConfigOpts() - try: - backend.register_opts(conf) - except cfg.DuplicateOptError: - pass - - conf.set_override('filesystem_store_datadir', - CONF.node_staging_uri[7:], - group='glance_store') - - # NOTE(flaper87): Do not even try to judge me for this... :( - # With the glance_store refactor, this code will change, until - # that happens, we don't have a better option and this is the - # least worst one, IMHO. - store = backend._load_store(conf, 'file') - - if store is None: - msg = (_("%(task_id)s of %(task_type)s not configured " - "properly. Could not load the filesystem store") % - {'task_id': self.task_id, 'task_type': self.task_type}) - raise exception.BadTaskConfiguration(msg) - - store.configure() - return store + super(_WebDownload, self).__init__(task_id, task_type, action_wrapper, + stores, 'WebDownload') def execute(self): """Create temp file into store and return path to it @@ -133,36 +71,6 @@ class _WebDownload(task.Task): pass return self._path - def revert(self, result, **kwargs): - if isinstance(result, failure.Failure): - LOG.error(_LE('Task: %(task_id)s failed to import image ' - '%(image_id)s to the filesystem.'), - {'task_id': self.task_id, - 'image_id': self.image_id}) - # NOTE(abhishekk): Revert image state back to 'queued' as - # something went wrong. - # NOTE(danms): If we failed to stage the image, then none - # of the _ImportToStore() tasks could have run, so we need - # to move all stores out of "importing" and into "failed". - with self.action_wrapper as action: - action.set_image_attribute(status='queued') - action.remove_importing_stores(self.stores) - action.add_failed_stores(self.stores) - - # NOTE(abhishekk): Deleting partial image data from staging area - if self._path is not None: - LOG.debug(('Deleting image %(image_id)s from staging ' - 'area.'), {'image_id': self.image_id}) - try: - if CONF.enabled_backends: - store_api.delete(self._path, None) - else: - store_api.delete_from_backend(self._path) - except Exception: - LOG.exception(_LE("Error reverting web-download " - "task: %(task_id)s"), { - 'task_id': self.task_id}) - def get_flow(**kwargs): """Return task flow for web-download. diff --git a/glance/async_/flows/api_image_import.py b/glance/async_/flows/api_image_import.py index 2f52d0ac28..ee72fbaeda 100644 --- a/glance/async_/flows/api_image_import.py +++ b/glance/async_/flows/api_image_import.py @@ -14,7 +14,9 @@ # under the License. import copy import functools +import json import os +import urllib.request import glance_store as store_api from glance_store import backend @@ -33,6 +35,7 @@ from taskflow import task from glance.api import common as api_common import glance.async_.flows._internal_plugins as internal_plugins import glance.async_.flows.plugins as import_plugins +from glance.async_ import utils from glance.common import exception from glance.common.scripts.image_import import main as image_import from glance.common.scripts import utils as script_utils @@ -79,6 +82,30 @@ Possible values: CONF.register_opts(api_import_opts, group='image_import_opts') +glance_download_opts = [ + cfg.ListOpt('extra_properties', + item_type=cfg.types.String(quotes=True), + bounds=True, + default=[ + 'hw_', 'trait:', 'os_distro', 'os_secure_boot', + 'os_type'], + help=_(""" +Specify metadata prefix to be set on the target image when using +glance-download. All other properties coming from the source image won't be set +on the target image. If specified metadata does not exist on the source image +it won't be set on the target image. Note you can't set the os_glance prefix +as it is reserved by glance, so the related properties won't be set on the +target image. + +Possible values: + * List containing extra_properties prefixes: ['os_', 'architecture'] + +""")), +] + +CONF.register_opts(glance_download_opts, group='glance_download_properties') + + # TODO(jokke): We should refactor the task implementations so that we do not # need to duplicate what we have already for example in base_import.py. @@ -89,6 +116,12 @@ class _NoStoresSucceeded(exception.GlanceException): super(_NoStoresSucceeded, self).__init__(message) +class _InvalidGlanceDownloadImageStatus(exception.GlanceException): + + def __init__(self, message): + super(_InvalidGlanceDownloadImageStatus, self).__init__(message) + + class ImportActionWrapper(object): """Wrapper for all the image metadata operations we do during an import. @@ -207,6 +240,18 @@ class _ImportActions(object): # should have moderated access like all the other things here. return copy.deepcopy(self._image.locations) + @property + def image_disk_format(self): + return self._image.disk_format + + @property + def image_container_format(self): + return self._image.container_format + + @property + def image_extra_properties(self): + return dict(self._image.extra_properties) + @property def image_status(self): return self._image.status @@ -739,6 +784,93 @@ class _CompleteTask(task.Task): {'task_id': self.task_id, 'task_type': self.task_type}) +class _ImportMetadata(task.Task): + + default_provides = 'image_size' + + def __init__(self, task_id, task_type, context, action_wrapper, + import_req): + self.task_id = task_id + self.task_type = task_type + self.context = context + self.action_wrapper = action_wrapper + self.import_req = import_req + self.props_to_copy = CONF.glance_download_properties.extra_properties + # We store the properties that will be set in case we are reverting + self.properties = {} + self.old_properties = {} + self.old_attributes = {} + super(_ImportMetadata, self).__init__( + name='%s-ImportMetdata-%s' % (task_type, task_id)) + + def execute(self): + try: + glance_endpoint = utils.get_glance_endpoint( + self.context, + self.import_req['method']['glance_region'], + self.import_req['method']['glance_service_interface']) + glance_image_id = self.import_req['method']['glance_image_id'] + image_download_metadata_url = '%s/v2/images/%s' % ( + glance_endpoint, glance_image_id) + LOG.info(_LI("Fetching glance image metadata from remote host %s"), + image_download_metadata_url) + token = self.context.auth_token + request = urllib.request.Request(image_download_metadata_url, + headers={'X-Auth-Token': token}) + with urllib.request.urlopen(request) as payload: + data = json.loads(payload.read().decode('utf-8')) + + if data.get('status') != 'active': + raise _InvalidGlanceDownloadImageStatus( + _('Source image status should be active instead of %s') + % data['status']) + + for key, value in data.items(): + for metadata in self.props_to_copy: + if key.startswith(metadata): + self.properties[key] = value + + with self.action_wrapper as action: + # Save the old properties in case we need to revert + self.old_properties = action.image_extra_properties + self.old_attributes = { + 'container_format': action.image_container_format, + 'disk_format': action.image_disk_format, + } + + # Set disk_format and container_format attributes + action.set_image_attribute( + disk_format=data['disk_format'], + container_format=data['container_format']) + + # Set extra propoerties + if self.properties: + action.set_image_extra_properties(self.properties) + try: + return int(data['size']) + except (ValueError, KeyError): + raise exception.ImportTaskError( + _('Size attribute of remote image %s could not be ' + 'determined.' % glance_image_id)) + except Exception as e: + with excutils.save_and_reraise_exception(): + LOG.error( + "Task %(task_id)s failed with exception %(error)s", { + "error": encodeutils.exception_to_unicode(e), + "task_id": self.task_id + }) + + def revert(self, result, **kwargs): + """Revert the extra properties set and set the image in queued""" + with self.action_wrapper as action: + for image_property in self.properties: + if image_property not in self.old_properties: + action.pop_extra_property(image_property) + action.set_image_extra_properties(self.old_properties) + action.set_image_attribute(status='queued', + **self.old_attributes) + + def assert_quota(context, task_repo, task_id, stores, action_wrapper, enforce_quota_fn, **enforce_kwargs): @@ -776,11 +908,13 @@ def get_flow(**kwargs): image_repo = kwargs.get('image_repo') admin_repo = kwargs.get('admin_repo') image_id = kwargs.get('image_id') - import_method = kwargs.get('import_req')['method']['name'] - uri = kwargs.get('import_req')['method'].get('uri') + import_req = kwargs.get('import_req') + import_method = import_req['method']['name'] + uri = import_req['method'].get('uri') stores = kwargs.get('backend', [None]) - all_stores_must_succeed = kwargs.get('import_req').get( + all_stores_must_succeed = import_req.get( 'all_stores_must_succeed', True) + context = kwargs.get('context') separator = '' if not CONF.enabled_backends and not CONF.node_staging_uri.endswith('/'): @@ -803,7 +937,10 @@ def get_flow(**kwargs): flow.add(_ImageLock(task_id, task_type, action_wrapper)) - if import_method in ['web-download', 'copy-image']: + if import_method in ['web-download', 'copy-image', 'glance-download']: + if import_method == 'glance-download': + flow.add(_ImportMetadata(task_id, task_type, + context, action_wrapper, import_req)) internal_plugin = internal_plugins.get_import_plugin(**kwargs) flow.add(internal_plugin) if CONF.enabled_backends: @@ -874,9 +1011,9 @@ def get_flow(**kwargs): stores, action_wrapper, ks_quota.enforce_image_size_total, delta=image_size) - elif import_method in ('copy-image', 'web-download'): - # The copy-image and web-download methods will use staging space to - # do their work, so check that quota. + elif import_method in ('copy-image', 'web-download', 'glance-download'): + # The copy-image, web-download and glance-download methods will use + # staging space to do their work, so check that quota. assert_quota(kwargs['context'], task_repo, task_id, stores, action_wrapper, ks_quota.enforce_image_staging_total, diff --git a/glance/common/config.py b/glance/common/config.py index 59091422b2..07345978a7 100644 --- a/glance/common/config.py +++ b/glance/common/config.py @@ -572,7 +572,7 @@ Related options: item_type=cfg.types.String(quotes=True), bounds=True, default=['glance-direct', 'web-download', - 'copy-image'], + 'copy-image', 'glance-download'], help=_(""" List of enabled Image Import Methods diff --git a/glance/tests/unit/async_/flows/test_api_image_import.py b/glance/tests/unit/async_/flows/test_api_image_import.py index 937c7484f4..65384555f3 100644 --- a/glance/tests/unit/async_/flows/test_api_image_import.py +++ b/glance/tests/unit/async_/flows/test_api_image_import.py @@ -15,6 +15,7 @@ import sys from unittest import mock +import urllib.error from glance_store import exceptions as store_exceptions from oslo_config import cfg @@ -25,6 +26,7 @@ import glance.async_.flows.api_image_import as import_flow from glance.common import exception from glance.common.scripts.image_import import main as image_import from glance import context +from glance.domain import ExtraProperties from glance import gateway import glance.tests.utils as test_utils @@ -890,7 +892,9 @@ class TestImportActions(test_utils.BaseTestCase): self.image = mock.MagicMock() self.image.image_id = IMAGE_ID1 self.image.status = 'active' - self.image.extra_properties = {'speed': '88mph'} + self.image.disk_format = 'raw' + self.image.container_format = 'bare' + self.image.extra_properties = ExtraProperties({'speed': '88mph'}) self.image.checksum = mock.sentinel.checksum self.image.os_hash_algo = mock.sentinel.hash_algo self.image.os_hash_value = mock.sentinel.hash_value @@ -900,6 +904,10 @@ class TestImportActions(test_utils.BaseTestCase): def test_image_property_proxies(self): self.assertEqual(IMAGE_ID1, self.actions.image_id) self.assertEqual('active', self.actions.image_status) + self.assertEqual('raw', self.actions.image_disk_format) + self.assertEqual('bare', self.actions.image_container_format) + self.assertEqual({'speed': '88mph'}, + self.actions.image_extra_properties) def test_merge_store_list(self): # Addition with no existing property works @@ -1131,3 +1139,152 @@ class TestCompleteTask(test_utils.BaseTestCase): {'image': IMAGE_ID1, 'task': TASK_ID1}) self.task.succeed.assert_called_once_with({'image_id': IMAGE_ID1}) + + +class TestImportMetadata(test_utils.BaseTestCase): + def setUp(self): + super(TestImportMetadata, self).setUp() + self.config(extra_properties=[], + group="glance_download_properties") + self.wrapper = mock.MagicMock(image_id=IMAGE_ID1) + self.context = context.RequestContext(user_id=TENANT1, + project_id=TENANT1, + overwrite=False) + self.import_req = { + 'method': { + 'glance_region': 'RegionTwo', + 'glance_service_interface': 'public', + 'glance_image_id': IMAGE_ID1 + } + } + + @mock.patch('urllib.request') + @mock.patch('glance.async_.flows.api_image_import.json') + @mock.patch('glance.async_.utils.get_glance_endpoint') + def test_execute_return_image_size(self, mock_gge, mock_json, + mock_request): + self.config(extra_properties=['hw:numa_nodes', 'os_hash'], + group="glance_download_properties") + mock_gge.return_value = 'https://other.cloud.foo/image' + action = self.wrapper.__enter__.return_value + mock_json.loads.return_value = { + 'status': 'active', + 'disk_format': 'qcow2', + 'container_format': 'bare', + 'hw:numa_nodes': '2', + 'os_hash': 'hash', + 'extra_metadata': 'hello', + 'size': '12345' + } + task = import_flow._ImportMetadata(TASK_ID1, TASK_TYPE, + self.context, self.wrapper, + self.import_req) + self.assertEqual(12345, task.execute()) + mock_request.Request.assert_called_once_with( + 'https://other.cloud.foo/image/v2/images/%s' % ( + IMAGE_ID1), + headers={'X-Auth-Token': self.context.auth_token}) + mock_gge.assert_called_once_with(self.context, 'RegionTwo', 'public') + action.set_image_attribute.assert_called_once_with( + disk_format='qcow2', + container_format='bare') + action.set_image_extra_properties.assert_called_once_with({ + 'hw:numa_nodes': '2', + 'os_hash': 'hash' + }) + + @mock.patch('urllib.request') + @mock.patch('glance.async_.utils.get_glance_endpoint') + def test_execute_fail_no_glance_endpoint(self, mock_gge, mock_request): + action = self.wrapper.__enter__.return_value + mock_gge.side_effect = exception.GlanceEndpointNotFound( + region='RegionTwo', + interface='public') + task = import_flow._ImportMetadata(TASK_ID1, TASK_TYPE, + self.context, self.wrapper, + self.import_req) + self.assertRaises(exception.GlanceEndpointNotFound, + task.execute) + action.assert_not_called() + mock_request.assert_not_called() + + @mock.patch('urllib.request') + @mock.patch('glance.async_.utils.get_glance_endpoint') + def test_execute_fail_remote_glance_unreachable(self, mock_gge, mock_r): + action = self.wrapper.__enter__.return_value + mock_r.urlopen.side_effect = urllib.error.HTTPError( + '/file', 400, 'Test Fail', {}, None) + task = import_flow._ImportMetadata(TASK_ID1, TASK_TYPE, + self.context, self.wrapper, + self.import_req) + self.assertRaises(urllib.error.HTTPError, + task.execute) + action.assert_not_called() + + @mock.patch('urllib.request') + @mock.patch('glance.async_.flows.api_image_import.json') + @mock.patch('glance.async_.utils.get_glance_endpoint') + def test_execute_invalid_remote_image_state(self, mock_gge, mock_json, + mock_request): + action = self.wrapper.__enter__.return_value + mock_gge.return_value = 'https://other.cloud.foo/image' + mock_json.loads.return_value = { + 'status': 'queued', + } + task = import_flow._ImportMetadata(TASK_ID1, TASK_TYPE, + self.context, self.wrapper, + self.import_req) + self.assertRaises(import_flow._InvalidGlanceDownloadImageStatus, + task.execute) + action.assert_not_called() + + @mock.patch('urllib.request') + @mock.patch('glance.async_.flows.api_image_import.json') + @mock.patch('glance.async_.utils.get_glance_endpoint') + def test_execute_raise_if_no_size(self, mock_gge, mock_json, mock_request): + self.config(extra_properties=['hw:numa_nodes', 'os_hash'], + group="glance_download_properties") + mock_gge.return_value = 'https://other.cloud.foo/image' + action = self.wrapper.__enter__.return_value + mock_json.loads.return_value = { + 'status': 'active', + 'disk_format': 'qcow2', + 'container_format': 'bare', + 'hw:numa_nodes': '2', + 'os_hash': 'hash', + 'extra_metadata': 'hello', + } + task = import_flow._ImportMetadata(TASK_ID1, TASK_TYPE, + self.context, self.wrapper, + self.import_req) + self.assertRaises(exception.ImportTaskError, task.execute) + mock_request.Request.assert_called_once_with( + 'https://other.cloud.foo/image/v2/images/%s' % ( + IMAGE_ID1), + headers={'X-Auth-Token': self.context.auth_token}) + mock_gge.assert_called_once_with(self.context, 'RegionTwo', 'public') + action.set_image_attribute.assert_called_once_with( + disk_format='qcow2', + container_format='bare') + action.set_image_extra_properties.assert_called_once_with({ + 'hw:numa_nodes': '2', + 'os_hash': 'hash' + }) + + def test_revert_rollback_metadata_value(self): + action = self.wrapper.__enter__.return_value + task = import_flow._ImportMetadata(TASK_ID1, TASK_TYPE, + self.context, self.wrapper, + self.import_req) + task.properties = {'prop1': 'value1', 'prop2': 'value2'} + task.old_properties = {'prop1': 'orig_val', 'old_prop': 'old_value'} + task.old_attributes = {'container_format': 'bare', + 'disk_format': 'qcow2'} + task.revert(None) + action.set_image_attribute.assert_called_once_with( + status='queued', + container_format='bare', + disk_format='qcow2') + action.pop_extra_property.assert_called_once_with('prop2') + action.set_image_extra_properties.assert_called_once_with( + task.old_properties) diff --git a/glance/tests/unit/async_/flows/test_base_download.py b/glance/tests/unit/async_/flows/test_base_download.py new file mode 100644 index 0000000000..32fbd5989a --- /dev/null +++ b/glance/tests/unit/async_/flows/test_base_download.py @@ -0,0 +1,207 @@ +# Copyright 2022 OVHCloud +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +from unittest import mock + +from glance_store import backend +from oslo_config import cfg +from taskflow.types import failure + +from glance.async_.flows import api_image_import +import glance.common.exception +from glance import domain +import glance.tests.unit.utils as unit_test_utils +import glance.tests.utils as test_utils + +CONF = cfg.CONF + + +TENANT1 = '6838eb7b-6ded-434a-882c-b344c77fe8df' + + +class TestBaseDownloadTask(test_utils.BaseTestCase): + + def setUp(self): + super(TestBaseDownloadTask, self).setUp() + + self.config(node_staging_uri='/tmp/staging') + self.image_repo = mock.MagicMock() + self.image_id = mock.MagicMock() + self.uri = mock.MagicMock() + self.plugin_name = 'FakeBaseDownload' + self.task_factory = domain.TaskFactory() + + task_input = { + "import_req": { + 'method': { + 'name': 'web_download', + 'uri': 'http://cloud.foo/image.qcow2' + } + } + } + task_ttl = CONF.task.task_time_to_live + + self.task_type = 'import' + request_id = 'fake_request_id' + user_id = 'fake_user' + self.task = self.task_factory.new_task(self.task_type, TENANT1, + self.image_id, user_id, + request_id, + task_time_to_live=task_ttl, + task_input=task_input) + + self.task_id = self.task.task_id + self.action_wrapper = api_image_import.ImportActionWrapper( + self.image_repo, self.image_id, self.task_id) + self.image_repo.get.return_value = mock.MagicMock( + extra_properties={'os_glance_import_task': self.task_id}) + self.base_download_task = unit_test_utils.FakeBaseDownloadPlugin( + self.task.task_id, self.task_type, self.action_wrapper, + ['foo'], self.plugin_name) + self.base_download_task._path = "/path/to_downloaded_data" + + def test_base_download_node_staging_uri_is_none(self): + self.config(node_staging_uri=None) + self.assertRaises(glance.common.exception.BadTaskConfiguration, + unit_test_utils.FakeBaseDownloadPlugin, + self.task.task_id, self.task_type, self.uri, + self.action_wrapper, ['foo']) + + @mock.patch.object(cfg.ConfigOpts, "set_override") + def test_base_download_node_store_initialization_failed( + self, mock_override): + with mock.patch.object(backend, '_load_store') as mock_load_store: + mock_load_store.return_value = None + self.assertRaises(glance.common.exception.BadTaskConfiguration, + unit_test_utils.FakeBaseDownloadPlugin, + self.task.task_id, self.task_type, self.uri, + self.action_wrapper, ['foo']) + mock_override.assert_called() + + def test_base_download_delete_staging_image_not_exist(self): + staging_path = "file:///tmp/staging/temp-image" + delete_from_fs_task = api_image_import._DeleteFromFS( + self.task.task_id, self.task_type) + with mock.patch.object(os.path, "exists") as mock_exists: + mock_exists.return_value = False + with mock.patch.object(os, "unlink") as mock_unlik: + delete_from_fs_task.execute(staging_path) + + self.assertEqual(1, mock_exists.call_count) + self.assertEqual(0, mock_unlik.call_count) + + @mock.patch.object(os.path, "exists") + def test_base_download_delete_staging_image_failed(self, mock_exists): + mock_exists.return_value = True + staging_path = "file:///tmp/staging/temp-image" + delete_from_fs_task = api_image_import._DeleteFromFS( + self.task.task_id, self.task_type) + with mock.patch.object(os, "unlink") as mock_unlink: + try: + delete_from_fs_task.execute(staging_path) + except OSError: + self.assertEqual(1, mock_unlink.call_count) + + self.assertEqual(1, mock_exists.call_count) + + @mock.patch.object(os.path, "exists") + def test_base_download_delete_staging_image_succeed(self, mock_exists): + mock_exists.return_value = True + staging_path = "file:///tmp/staging/temp-image" + delete_from_fs_task = api_image_import._DeleteFromFS( + self.task.task_id, self.task_type) + with mock.patch.object(os, "unlink") as mock_unlik: + delete_from_fs_task.execute(staging_path) + self.assertEqual(1, mock_exists.call_count) + self.assertEqual(1, mock_unlik.call_count) + + @mock.patch( + "glance.async_.flows._internal_plugins.base_download.store_api") + def test_base_download_revert_with_failure(self, mock_store_api): + image = self.image_repo.get.return_value + image.extra_properties['os_glance_importing_to_stores'] = 'foo' + image.extra_properties['os_glance_failed_import'] = '' + + self.base_download_task.execute = mock.MagicMock( + side_effect=glance.common.exception.ImportTaskError) + self.base_download_task.revert(None) + mock_store_api.delete_from_backend.assert_called_once_with( + "/path/to_downloaded_data") + self.assertEqual(1, self.image_repo.save.call_count) + self.assertEqual( + '', image.extra_properties['os_glance_importing_to_stores']) + self.assertEqual( + 'foo', image.extra_properties['os_glance_failed_import']) + + @mock.patch( + "glance.async_.flows._internal_plugins.base_download.store_api") + def test_base_download_revert_without_failure_multi_store(self, + mock_store_api): + enabled_backends = { + 'fast': 'file', + 'cheap': 'file' + } + self.config(enabled_backends=enabled_backends) + + self.base_download_task.revert("/path/to_downloaded_data") + mock_store_api.delete.assert_called_once_with( + "/path/to_downloaded_data", None) + + @mock.patch( + "glance.async_.flows._internal_plugins.base_download.store_api") + def test_base_download_revert_with_failure_without_path(self, + mock_store_api): + image = self.image_repo.get.return_value + image.status = 'importing' + image.extra_properties['os_glance_importing_to_stores'] = 'foo' + image.extra_properties['os_glance_failed_import'] = '' + result = failure.Failure.from_exception( + glance.common.exception.ImportTaskError()) + + self.base_download_task._path = None + self.base_download_task.revert(result) + mock_store_api.delete_from_backend.assert_not_called() + + # NOTE(danms): Since we told revert that we were the problem, + # we should have updated the image status and moved the stores + # to the failed list. + self.image_repo.save.assert_called_once_with(image, 'importing') + self.assertEqual('queued', image.status) + self.assertEqual( + '', image.extra_properties['os_glance_importing_to_stores']) + self.assertEqual( + 'foo', image.extra_properties['os_glance_failed_import']) + + @mock.patch( + "glance.async_.flows._internal_plugins.base_download.store_api") + def test_base_download_revert_with_failure_with_path(self, mock_store_api): + result = failure.Failure.from_exception( + glance.common.exception.ImportTaskError()) + + self.base_download_task.revert(result) + mock_store_api.delete_from_backend.assert_called_once_with( + "/path/to_downloaded_data") + + @mock.patch( + "glance.async_.flows._internal_plugins.base_download.store_api") + def test_base_download_delete_fails_on_revert(self, mock_store_api): + result = failure.Failure.from_exception( + glance.common.exception.ImportTaskError()) + mock_store_api.delete_from_backend.side_effect = Exception + + # this will verify that revert does not break because of failure + # while deleting data in staging area + self.base_download_task.revert(result) diff --git a/glance/tests/unit/async_/flows/test_glance_download.py b/glance/tests/unit/async_/flows/test_glance_download.py new file mode 100644 index 0000000000..5a391e50d9 --- /dev/null +++ b/glance/tests/unit/async_/flows/test_glance_download.py @@ -0,0 +1,167 @@ +# Copyright 2022 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from unittest import mock +import urllib.error + +from glance_store._drivers import filesystem +from oslo_config import cfg +from oslo_utils.fixture import uuidsentinel + +from glance.async_.flows._internal_plugins import glance_download +from glance.async_.flows import api_image_import +import glance.common.exception +import glance.context +from glance import domain +import glance.tests.utils as test_utils + +CONF = cfg.CONF + + +TENANT1 = '6838eb7b-6ded-434a-882c-b344c77fe8df' + + +class TestGlanceDownloadTask(test_utils.BaseTestCase): + + def setUp(self): + super(TestGlanceDownloadTask, self).setUp() + + self.config(node_staging_uri='/tmp/staging') + self.image_repo = mock.MagicMock() + self.image_id = mock.MagicMock() + self.uri = mock.MagicMock() + self.task_factory = domain.TaskFactory() + self.context = glance.context.RequestContext(tenant=TENANT1, + auth_token='token') + task_input = { + "import_req": { + 'method': { + 'name': 'glance-download', + 'glance_image_id': uuidsentinel.remote_image, + 'glance_region': 'RegionTwo', + 'glance_service_interface': 'public', + } + } + } + task_ttl = CONF.task.task_time_to_live + + self.task_type = 'import' + request_id = 'fake_request_id' + user_id = 'fake_user' + self.task = self.task_factory.new_task(self.task_type, TENANT1, + self.image_id, user_id, + request_id, + task_time_to_live=task_ttl, + task_input=task_input) + + self.task_id = self.task.task_id + self.action_wrapper = api_image_import.ImportActionWrapper( + self.image_repo, self.image_id, self.task_id) + self.image_repo.get.return_value = mock.MagicMock( + extra_properties={'os_glance_import_task': self.task_id}) + + @mock.patch.object(filesystem.Store, 'add') + @mock.patch('glance.async_.utils.get_glance_endpoint') + def test_glance_download(self, mock_gge, mock_add): + mock_gge.return_value = 'https://other.cloud.foo/image' + glance_download_task = glance_download._DownloadGlanceImage( + self.context, self.task.task_id, self.task_type, + self.action_wrapper, ['foo'], + 'RegionTwo', uuidsentinel.remote_image, 'public') + with mock.patch('urllib.request') as mock_request: + mock_add.return_value = ["path", 12345] + self.assertEqual(glance_download_task.execute(12345), "path") + mock_add.assert_called_once_with( + self.image_id, + mock_request.urlopen.return_value, 0) + mock_request.Request.assert_called_once_with( + 'https://other.cloud.foo/image/v2/images/%s/file' % ( + uuidsentinel.remote_image), + headers={'X-Auth-Token': self.context.auth_token}) + mock_gge.assert_called_once_with(self.context, 'RegionTwo', 'public') + + @mock.patch.object(filesystem.Store, 'add') + @mock.patch('glance.async_.utils.get_glance_endpoint') + def test_glance_download_failed(self, mock_gge, mock_add): + mock_gge.return_value = 'https://other.cloud.foo/image' + glance_download_task = glance_download._DownloadGlanceImage( + self.context, self.task.task_id, self.task_type, + self.action_wrapper, ['foo'], + 'RegionTwo', uuidsentinel.remote_image, 'public') + with mock.patch('urllib.request') as mock_request: + mock_request.urlopen.side_effect = urllib.error.HTTPError( + '/file', 400, 'Test Fail', {}, None) + self.assertRaises(urllib.error.HTTPError, + glance_download_task.execute, + 12345) + mock_add.assert_not_called() + mock_request.Request.assert_called_once_with( + 'https://other.cloud.foo/image/v2/images/%s/file' % ( + uuidsentinel.remote_image), + headers={'X-Auth-Token': self.context.auth_token}) + mock_gge.assert_called_once_with(self.context, 'RegionTwo', 'public') + + @mock.patch('urllib.request') + @mock.patch('glance.async_.utils.get_glance_endpoint') + def test_glance_download_no_glance_endpoint(self, mock_gge, mock_request): + mock_gge.side_effect = glance.common.exception.GlanceEndpointNotFound( + region='RegionTwo', + interface='public') + glance_download_task = glance_download._DownloadGlanceImage( + self.context, self.task.task_id, self.task_type, + self.action_wrapper, ['foo'], + 'RegionTwo', uuidsentinel.remote_image, 'public') + self.assertRaises(glance.common.exception.GlanceEndpointNotFound, + glance_download_task.execute, 12345) + mock_request.assert_not_called() + + @mock.patch.object(filesystem.Store, 'add') + @mock.patch('glance.async_.utils.get_glance_endpoint') + def test_glance_download_size_mismatch(self, mock_gge, mock_add): + mock_gge.return_value = 'https://other.cloud.foo/image' + glance_download_task = glance_download._DownloadGlanceImage( + self.context, self.task.task_id, self.task_type, + self.action_wrapper, ['foo'], + 'RegionTwo', uuidsentinel.remote_image, 'public') + with mock.patch('urllib.request') as mock_request: + mock_add.return_value = ["path", 1] + self.assertRaises(glance.common.exception.ImportTaskError, + glance_download_task.execute, 12345) + mock_add.assert_called_once_with( + self.image_id, + mock_request.urlopen.return_value, 0) + mock_request.Request.assert_called_once_with( + 'https://other.cloud.foo/image/v2/images/%s/file' % ( + uuidsentinel.remote_image), + headers={'X-Auth-Token': self.context.auth_token}) + mock_gge.assert_called_once_with(self.context, 'RegionTwo', 'public') + + @mock.patch('urllib.request') + @mock.patch('glance.common.utils.validate_import_uri') + @mock.patch('glance.async_.utils.get_glance_endpoint') + def test_glance_download_wrong_download_url(self, mock_gge, mock_validate, + mock_request): + mock_validate.return_value = False + mock_gge.return_value = 'https://other.cloud.foo/image' + glance_download_task = glance_download._DownloadGlanceImage( + self.context, self.task.task_id, self.task_type, + self.action_wrapper, ['foo'], + 'RegionTwo', uuidsentinel.remote_image, 'public') + self.assertRaises(glance.common.exception.ImportTaskError, + glance_download_task.execute, 12345) + mock_request.assert_not_called() + mock_validate.assert_called_once_with( + 'https://other.cloud.foo/image/v2/images/%s/file' % ( + uuidsentinel.remote_image)) diff --git a/glance/tests/unit/async_/flows/test_web_download.py b/glance/tests/unit/async_/flows/test_web_download.py index c03992d874..997fcf90a7 100644 --- a/glance/tests/unit/async_/flows/test_web_download.py +++ b/glance/tests/unit/async_/flows/test_web_download.py @@ -13,13 +13,10 @@ # License for the specific language governing permissions and limitations # under the License. -import os from unittest import mock from glance_store._drivers import filesystem -from glance_store import backend from oslo_config import cfg -from taskflow.types import failure from glance.async_.flows._internal_plugins import web_download from glance.async_.flows import api_image_import @@ -67,217 +64,84 @@ class TestWebDownloadTask(test_utils.BaseTestCase): self.task_id = self.task.task_id self.action_wrapper = api_image_import.ImportActionWrapper( self.image_repo, self.image_id, self.task_id) + self.web_download_task = web_download._WebDownload( + self.task.task_id, self.task_type, self.uri, self.action_wrapper, + ['foo']) self.image_repo.get.return_value = mock.MagicMock( extra_properties={'os_glance_import_task': self.task_id}) @mock.patch.object(filesystem.Store, 'add') def test_web_download(self, mock_add): - web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.uri, self.action_wrapper, - ['foo']) with mock.patch.object(script_utils, 'get_image_data_iter') as mock_iter: mock_add.return_value = ["path", 4] mock_iter.return_value.headers = {} - self.assertEqual(web_download_task.execute(), "path") + self.assertEqual(self.web_download_task.execute(), "path") mock_add.assert_called_once_with(self.image_id, mock_iter.return_value, 0) @mock.patch.object(filesystem.Store, 'add') def test_web_download_with_content_length(self, mock_add): - web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.uri, self.action_wrapper, - ['foo']) with mock.patch.object(script_utils, 'get_image_data_iter') as mock_iter: mock_iter.return_value.headers = {'content-length': '4'} mock_add.return_value = ["path", 4] - self.assertEqual(web_download_task.execute(), "path") + self.assertEqual(self.web_download_task.execute(), "path") mock_add.assert_called_once_with(self.image_id, mock_iter.return_value, 0) @mock.patch.object(filesystem.Store, 'add') def test_web_download_with_invalid_content_length(self, mock_add): - web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.uri, self.action_wrapper, - ['foo']) with mock.patch.object(script_utils, 'get_image_data_iter') as mock_iter: mock_iter.return_value.headers = {'content-length': "not_valid"} mock_add.return_value = ["path", 4] - self.assertEqual(web_download_task.execute(), "path") + self.assertEqual(self.web_download_task.execute(), "path") mock_add.assert_called_once_with(self.image_id, mock_iter.return_value, 0) @mock.patch.object(filesystem.Store, 'add') def test_web_download_fails_when_data_size_different(self, mock_add): - web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.uri, self.action_wrapper, - ['foo']) with mock.patch.object(script_utils, 'get_image_data_iter') as mock_iter: mock_iter.return_value.headers = {'content-length': '4'} mock_add.return_value = ["path", 3] self.assertRaises( glance.common.exception.ImportTaskError, - web_download_task.execute) - - def test_web_download_node_staging_uri_is_none(self): - self.config(node_staging_uri=None) - self.assertRaises(glance.common.exception.BadTaskConfiguration, - web_download._WebDownload, self.task.task_id, - self.task_type, self.uri, self.action_wrapper, - ['foo']) - - @mock.patch.object(cfg.ConfigOpts, "set_override") - def test_web_download_node_store_initialization_failed(self, - mock_override): - with mock.patch.object(backend, '_load_store') as mock_load_store: - mock_load_store.return_value = None - self.assertRaises(glance.common.exception.BadTaskConfiguration, - web_download._WebDownload, self.task.task_id, - self.task_type, self.uri, self.action_wrapper, - ['foo']) - mock_override.assert_called() + self.web_download_task.execute) def test_web_download_failed(self): - web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.uri, self.action_wrapper, - ['foo']) with mock.patch.object(script_utils, "get_image_data_iter") as mock_iter: mock_iter.side_effect = glance.common.exception.NotFound self.assertRaises(glance.common.exception.NotFound, - web_download_task.execute) - - def test_web_download_delete_staging_image_not_exist(self): - staging_path = "file:///tmp/staging/temp-image" - delete_from_fs_task = api_image_import._DeleteFromFS( - self.task.task_id, self.task_type) - with mock.patch.object(os.path, "exists") as mock_exists: - mock_exists.return_value = False - with mock.patch.object(os, "unlink") as mock_unlik: - delete_from_fs_task.execute(staging_path) - - self.assertEqual(1, mock_exists.call_count) - self.assertEqual(0, mock_unlik.call_count) - - @mock.patch.object(os.path, "exists") - def test_web_download_delete_staging_image_failed(self, mock_exists): - mock_exists.return_value = True - staging_path = "file:///tmp/staging/temp-image" - delete_from_fs_task = api_image_import._DeleteFromFS( - self.task.task_id, self.task_type) - with mock.patch.object(os, "unlink") as mock_unlink: - try: - delete_from_fs_task.execute(staging_path) - except OSError: - self.assertEqual(1, mock_unlink.call_count) - - self.assertEqual(1, mock_exists.call_count) - - @mock.patch.object(os.path, "exists") - def test_web_download_delete_staging_image_succeed(self, mock_exists): - mock_exists.return_value = True - staging_path = "file:///tmp/staging/temp-image" - delete_from_fs_task = api_image_import._DeleteFromFS( - self.task.task_id, self.task_type) - with mock.patch.object(os, "unlink") as mock_unlik: - delete_from_fs_task.execute(staging_path) - self.assertEqual(1, mock_exists.call_count) - self.assertEqual(1, mock_unlik.call_count) + self.web_download_task.execute) @mock.patch.object(filesystem.Store, 'add') - @mock.patch("glance.async_.flows._internal_plugins.web_download.store_api") - def test_web_download_revert_with_failure(self, mock_store_api, - mock_add): - image = self.image_repo.get.return_value - image.extra_properties['os_glance_importing_to_stores'] = 'foo' - image.extra_properties['os_glance_failed_import'] = '' - web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.uri, self.action_wrapper, - ['foo']) + def test_web_download_check_content_length(self, mock_add): with mock.patch.object(script_utils, 'get_image_data_iter') as mock_iter: + mock_add.return_value = ["path", 4] mock_iter.return_value.headers = {'content-length': '4'} - mock_add.return_value = "/path/to_downloaded_data", 3 - self.assertRaises( - glance.common.exception.ImportTaskError, - web_download_task.execute) + self.assertEqual(self.web_download_task.execute(), "path") + mock_add.assert_called_once_with(self.image_id, + mock_iter.return_value, 0) - web_download_task.revert(None) - mock_store_api.delete_from_backend.assert_called_once_with( - "/path/to_downloaded_data") - # NOTE(danms): Since we told revert that we were not at fault, - # we should not have updated the image. - self.image_repo.save.assert_not_called() - self.assertEqual( - 'foo', image.extra_properties['os_glance_importing_to_stores']) - self.assertEqual( - '', image.extra_properties['os_glance_failed_import']) + @mock.patch.object(filesystem.Store, 'add') + def test_web_download_invalid_content_length(self, mock_add): + with mock.patch.object(script_utils, + 'get_image_data_iter') as mock_iter: + mock_add.return_value = ["path", 4] + mock_iter.return_value.headers = {'content-length': 'not_valid'} + self.assertEqual(self.web_download_task.execute(), "path") + mock_add.assert_called_once_with(self.image_id, + mock_iter.return_value, 0) - @mock.patch("glance.async_.flows._internal_plugins.web_download.store_api") - def test_web_download_revert_without_failure_multi_store(self, - mock_store_api): - enabled_backends = { - 'fast': 'file', - 'cheap': 'file' - } - self.config(enabled_backends=enabled_backends) - web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.uri, self.action_wrapper, - ['foo']) - web_download_task._path = "/path/to_downloaded_data" - web_download_task.revert("/path/to_downloaded_data") - mock_store_api.delete.assert_called_once_with( - "/path/to_downloaded_data", None) - - @mock.patch("glance.async_.flows._internal_plugins.web_download.store_api") - def test_web_download_revert_with_failure_without_path(self, - mock_store_api): - image = self.image_repo.get.return_value - image.status = 'importing' - image.extra_properties['os_glance_importing_to_stores'] = 'foo' - image.extra_properties['os_glance_failed_import'] = '' - result = failure.Failure.from_exception( - glance.common.exception.ImportTaskError()) - web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.uri, self.action_wrapper, - ['foo']) - web_download_task.revert(result) - mock_store_api.delete_from_backend.assert_not_called() - - # NOTE(danms): Since we told revert that we were the problem, - # we should have updated the image status and moved the stores - # to the failed list. - self.image_repo.save.assert_called_once_with(image, 'importing') - self.assertEqual('queued', image.status) - self.assertEqual( - '', image.extra_properties['os_glance_importing_to_stores']) - self.assertEqual( - 'foo', image.extra_properties['os_glance_failed_import']) - - @mock.patch("glance.async_.flows._internal_plugins.web_download.store_api") - def test_web_download_revert_with_failure_with_path(self, mock_store_api): - result = failure.Failure.from_exception( - glance.common.exception.ImportTaskError()) - web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.uri, self.action_wrapper, - ['foo']) - web_download_task._path = "/path/to_downloaded_data" - web_download_task.revert(result) - mock_store_api.delete_from_backend.assert_called_once_with( - "/path/to_downloaded_data") - - @mock.patch("glance.async_.flows._internal_plugins.web_download.store_api") - def test_web_download_delete_fails_on_revert(self, mock_store_api): - result = failure.Failure.from_exception( - glance.common.exception.ImportTaskError()) - mock_store_api.delete_from_backend.side_effect = Exception - web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.uri, self.action_wrapper, - ['foo']) - web_download_task._path = "/path/to_downloaded_data" - # this will verify that revert does not break because of failure - # while deleting data in staging area - web_download_task.revert(result) + @mock.patch.object(filesystem.Store, 'add') + def test_web_download_wrong_content_length(self, mock_add): + with mock.patch.object(script_utils, + 'get_image_data_iter') as mock_iter: + mock_add.return_value = ["path", 2] + mock_iter.return_value.headers = {'content-length': '4'} + self.assertRaises(glance.common.exception.ImportTaskError, + self.web_download_task.execute) diff --git a/glance/tests/unit/utils.py b/glance/tests/unit/utils.py index ccf5107b33..3cbb51dbc2 100644 --- a/glance/tests/unit/utils.py +++ b/glance/tests/unit/utils.py @@ -21,6 +21,7 @@ import urllib from oslo_config import cfg +from glance.async_.flows._internal_plugins import base_download from glance.common import exception from glance.common import store_utils from glance.common import wsgi @@ -373,3 +374,8 @@ class FakeTask(object): def fail(self, message): self.message = message self._status = 'failure' + + +class FakeBaseDownloadPlugin(base_download.BaseDownload): + def execute(self): + pass diff --git a/glance/tests/unit/v2/test_discovery_image_import.py b/glance/tests/unit/v2/test_discovery_image_import.py index 14b68fbd84..949da8c077 100644 --- a/glance/tests/unit/v2/test_discovery_image_import.py +++ b/glance/tests/unit/v2/test_discovery_image_import.py @@ -36,7 +36,7 @@ class TestInfoControllers(test_utils.BaseTestCase): # TODO(rosmaita): change this when import methods are # listed in the config file import_methods = ['glance-direct', 'web-download', - 'copy-image'] + 'copy-image', 'glance-download'] req = unit_test_utils.get_fake_request() output = self.controller.get_image_import(req) diff --git a/glance/tests/unit/v2/test_images_resource.py b/glance/tests/unit/v2/test_images_resource.py index 67a6f36492..203092554a 100644 --- a/glance/tests/unit/v2/test_images_resource.py +++ b/glance/tests/unit/v2/test_images_resource.py @@ -911,6 +911,45 @@ class TestImagesController(base.IsolatedUnitTest): {'method': {'name': 'web-download', 'uri': 'fake_uri'}}) + def test_image_import_raises_bad_request_for_glance_download_missing_input( + self): + request = unit_test_utils.get_fake_request() + with mock.patch.object( + glance.notifier.ImageRepoProxy, 'get') as mock_get: + mock_get.return_value = FakeImage(status='queued') + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.import_image, request, UUID4, + {'method': {'name': 'glance-download'}}) + + def test_image_import_raise_bad_request_wrong_id_for_glance_download( + self): + request = unit_test_utils.get_fake_request() + with mock.patch.object( + glance.notifier.ImageRepoProxy, 'get') as mock_get: + mock_get.return_value = FakeImage(status='queued') + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.import_image, request, UUID4, + {'method': {'name': 'glance-download', + 'glance_image_id': 'fake_id', + 'glance_region': 'REGION4'}}) + + @mock.patch.object(glance.domain.TaskFactory, 'new_task') + @mock.patch.object(glance.notifier.ImageRepoProxy, 'get') + def test_image_import_add_default_service_endpoint_for_glance_download( + self, mock_get, mock_nt): + request = unit_test_utils.get_fake_request() + mock_get.return_value = FakeImage(status='queued') + body = {'method': {'name': 'glance-download', + 'glance_image_id': UUID4, + 'glance_region': 'REGION2'}} + self.controller.import_image(request, UUID4, body) + expected_req = {'method': {'name': 'glance-download', + 'glance_image_id': UUID4, + 'glance_region': 'REGION2', + 'glance_service_interface': 'public'}} + self.assertEqual(expected_req, + mock_nt.call_args.kwargs['task_input']['import_req']) + @mock.patch('glance.context.get_ksa_client') def test_image_import_proxies(self, mock_client): # Make sure that we proxy to the remote side when we need to @@ -4865,7 +4904,7 @@ class TestImagesDeserializer(test_utils.BaseTestCase): request.body = jsonutils.dump_as_bytes(import_body) return request - KNOWN_IMPORT_METHODS = ['glance-direct', 'web-download'] + KNOWN_IMPORT_METHODS = ['glance-direct', 'web-download', 'glance-download'] def test_import_image_invalid_import_method(self): # Bug 1754634: make sure that what's considered valid diff --git a/releasenotes/notes/add-glance-download-method-be6d9e927b8b0a43.yaml b/releasenotes/notes/add-glance-download-method-be6d9e927b8b0a43.yaml new file mode 100644 index 0000000000..7dcb1e79fa --- /dev/null +++ b/releasenotes/notes/add-glance-download-method-be6d9e927b8b0a43.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Glance to glance image import plugin. With this release users can import + an image from an other glance server from an other opensatck region. + The two glance services must use the same keystone service. The feature + use the same keystone authentication token on both glance services and + copy by default container_format, disk_format and customizable properties + from source image ``['hw_', 'trait:', 'os_distro', 'os_secure_boot', + 'os_type']`` diff --git a/setup.cfg b/setup.cfg index 611f5adc2c..ae28080563 100644 --- a/setup.cfg +++ b/setup.cfg @@ -82,6 +82,7 @@ glance.image_import.plugins = glance.image_import.internal_plugins = web_download = glance.async_.flows._internal_plugins.web_download:get_flow copy_image = glance.async_.flows._internal_plugins.copy_image:get_flow + glance_download = glance.async_.flows._internal_plugins.glance_download:get_flow [egg_info]