openstack/glance
Code Issues Proposed changes

Implement glance-download internal plugin

Browse Source 
Add a new import method called glance-download
that implements a glance to glance download in
a multi-region cloud with a federated Keystone.

This method will copy the image data and
selected metadata to the target glance, checking
that the downloaded size match the "size" image
attribute in the source glance.

Implements: blueprint glance-download-import
Co-Authored-By: Victor Coutellier <victor.coutellier@gmail.com>
Change-Id: Ic51c5fd87caf04d38aeaf758ad2d0e2f28098e4d
This commit is contained in:
Pierre-Samuel Le Stang
2022-05-03 16:47:54 +02:00
committed by Dan Smith
parent 653d52e90d
commit 480ea3825f
15 changed files with 1039 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
glance/api/v2/images.py
View File

@ -18,6 +18,7 @@ import http.client as http
import os
import re
import urllib.parse as urlparse
import uuid
from castellan.common import exception as castellan_exception
from castellan import key_manager
@ -332,9 +333,10 @@ class ImagesController(object):
msg = _("Only images with status active can be targeted for "
"copying")
raise exception.Conflict(msg)
if image.status != 'queued' and import_method == 'web-download':
if (image.status != 'queued' and
import_method in ['web-download', 'glance-download']):
msg = _("Image needs to be in 'queued' state to use "
"'web-download' method")
"'%s' method") % import_method
raise exception.Conflict(msg)
if (image.status != 'uploading' and
import_method == 'glance-direct'):
@ -347,6 +349,23 @@ class ImagesController(object):
if not getattr(image, 'disk_format', None):
msg = _("'disk_format' needs to be set before import")
raise exception.Conflict(msg)
if import_method == 'glance-download':
if 'glance_region' not in body.get('method'):
msg = _("'glance_region' needs to be set for "
"glance-download import method")
raise webob.exc.HTTPBadRequest(explanation=msg)
if 'glance_image_id' not in body.get('method'):
msg = _("'glance_image_id' needs to be set for "
"glance-download import method")
raise webob.exc.HTTPBadRequest(explanation=msg)
try:
uuid.UUID(body['method']['glance_image_id'])
except ValueError:
msg = (_("Remote image id does not look like a UUID: %s")
% body['method']['glance_image_id'])
raise webob.exc.HTTPBadRequest(explanation=msg)
if 'glance_service_interface' not in body.get('method'):
body.get('method')['glance_service_interface'] = 'public'
# NOTE(danms): For copy-image only, we check policy to decide
# if the user should be able to do this. Otherwise, we forbid

131
glance/async_/flows/_internal_plugins/base_download.py Normal file
View File

@ -0,0 +1,131 @@
# Copyright 2018 Red Hat, Inc.
# Copyright 2022 OVHCloud
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import glance_store as store_api
from glance_store import backend
from oslo_config import cfg
from oslo_log import log as logging
import six
from taskflow import task
from glance.common import exception
from glance.i18n import _, _LE
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
@six.add_metaclass(abc.ABCMeta)
class BaseDownload(task.Task):
default_provides = 'file_uri'
def __init__(self, task_id, task_type, action_wrapper, stores,
plugin_name):
self.task_id = task_id
self.task_type = task_type
self.image_id = action_wrapper.image_id
self.action_wrapper = action_wrapper
self.stores = stores
self._path = None
self.plugin_name = plugin_name or 'Download'
super(BaseDownload, self).__init__(
name='%s-%s-%s' % (task_type, self.plugin_name, task_id))
# NOTE(abhishekk): Use reserved 'os_glance_staging_store' for
# staging the data, the else part will be removed once old way
# of configuring store is deprecated.
if CONF.enabled_backends:
self.store = store_api.get_store_from_store_identifier(
'os_glance_staging_store')
else:
if CONF.node_staging_uri is None:
msg = (_("%(task_id)s of %(task_type)s not configured "
"properly. Missing node_staging_uri: %(work_dir)s") %
{'task_id': self.task_id,
'task_type': self.task_type,
'work_dir': CONF.node_staging_uri})
raise exception.BadTaskConfiguration(msg)
self.store = self._build_store()
def _build_store(self):
# NOTE(flaper87): Due to the nice glance_store api (#sarcasm), we're
# forced to build our own config object, register the required options
# (and by required I mean *ALL* of them, even the ones we don't want),
# and create our own store instance by calling a private function.
# This is certainly unfortunate but it's the best we can do until the
# glance_store refactor is done. A good thing is that glance_store is
# under our team's management and it gates on Glance so changes to
# this API will (should?) break task's tests.
# TODO(abhishekk): After removal of backend module from glance_store
# need to change this to use multi_backend module.
conf = cfg.ConfigOpts()
try:
backend.register_opts(conf)
except cfg.DuplicateOptError:
pass
conf.set_override('filesystem_store_datadir',
CONF.node_staging_uri[7:],
group='glance_store')
# NOTE(flaper87): Do not even try to judge me for this... :(
# With the glance_store refactor, this code will change, until
# that happens, we don't have a better option and this is the
# least worst one, IMHO.
store = store_api.backend._load_store(conf, 'file')
if store is None:
msg = (_("%(task_id)s of %(task_type)s not configured "
"properly. Could not load the filesystem store") %
{'task_id': self.task_id, 'task_type': self.task_type})
raise exception.BadTaskConfiguration(msg)
store.configure()
return store
def revert(self, result, **kwargs):
LOG.error(_LE('Task: %(task_id)s failed to import image '
'%(image_id)s to the filesystem.'),
{'task_id': self.task_id,
'image_id': self.image_id})
# NOTE(abhishekk): Revert image state back to 'queued' as
# something went wrong.
# NOTE(danms): If we failed to stage the image, then none
# of the _ImportToStore() tasks could have run, so we need
# to move all stores out of "importing" and into "failed".
with self.action_wrapper as action:
action.set_image_attribute(status='queued')
action.remove_importing_stores(self.stores)
action.add_failed_stores(self.stores)
# NOTE(abhishekk): Deleting partial image data from staging area
if self._path is not None:
LOG.debug(('Deleting image %(image_id)s from staging '
'area.'), {'image_id': self.image_id})
try:
if CONF.enabled_backends:
store_api.delete(self._path, None)
else:
store_api.delete_from_backend(self._path)
except Exception:
LOG.exception(_LE("Error reverting web/glance download "
"task: %(task_id)s"), {
'task_id': self.task_id})

115
glance/async_/flows/_internal_plugins/glance_download.py Normal file
View File

@ -0,0 +1,115 @@
# Copyright 2022 OVHCloud
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import urllib.request
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import encodeutils
from oslo_utils import excutils
from taskflow.patterns import linear_flow as lf
from glance.async_.flows._internal_plugins import base_download
from glance.async_ import utils
from glance.common import exception
from glance.common import utils as common_utils
from glance.i18n import _, _LI, _LE
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class _DownloadGlanceImage(base_download.BaseDownload):
def __init__(self, context, task_id, task_type, action_wrapper, stores,
glance_region, glance_image_id, glance_service_interface):
self.context = context
self.glance_region = glance_region
self.glance_image_id = glance_image_id
self.glance_service_interface = glance_service_interface
super(_DownloadGlanceImage,
self).__init__(task_id, task_type, action_wrapper, stores,
'GlanceDownload')
def execute(self, image_size):
"""Create temp file into store and return path to it
:param image_size: Glance Image Size retrieved from ImportMetadata task
"""
try:
glance_endpoint = utils.get_glance_endpoint(
self.context,
self.glance_region,
self.glance_service_interface)
image_download_url = '%s/v2/images/%s/file' % (
glance_endpoint, self.glance_image_id)
if not common_utils.validate_import_uri(image_download_url):
LOG.debug("Processed URI for glance-download does not pass "
"filtering: %s", image_download_url)
msg = (_("Processed URI for glance-download does not pass "
"filtering: %s") % image_download_url)
raise exception.ImportTaskError(msg)
LOG.info(_LI("Downloading glance image %s"), image_download_url)
token = self.context.auth_token
request = urllib.request.Request(image_download_url,
headers={'X-Auth-Token': token})
data = urllib.request.urlopen(request)
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.error(
_LE("Task %(task_id)s failed with exception %(error)s"), {
"error": encodeutils.exception_to_unicode(e),
"task_id": self.task_id
})
self._path, bytes_written = self.store.add(self.image_id, data, 0)[0:2]
if bytes_written != image_size:
msg = (_("Task %(task_id)s failed because downloaded data "
"size %(data_size)i is different from expected %("
"expected)i") %
{"task_id": self.task_id, "data_size": bytes_written,
"expected": image_size})
raise exception.ImportTaskError(msg)
return self._path
def get_flow(**kwargs):
"""Return task flow for no-op.
:param context: request context
:param task_id: Task ID.
:param task_type: Type of the task.
:param image_repo: Image repository used.
:param image_id: Image ID
:param source_region: Source region name
"""
context = kwargs.get('context')
task_id = kwargs.get('task_id')
task_type = kwargs.get('task_type')
action_wrapper = kwargs.get('action_wrapper')
stores = kwargs.get('backend', [None])
# glance-download parameters
import_req = kwargs.get('import_req')
method = import_req.get('method')
glance_region = method.get('glance_region')
glance_image_id = method.get('glance_image_id')
glance_service_interface = method.get('glance_service_interface')
return lf.Flow(task_type).add(
_DownloadGlanceImage(context, task_id, task_type, action_wrapper,
stores, glance_region, glance_image_id,
glance_service_interface),
)

106
glance/async_/flows/_internal_plugins/web_download.py
View File

@ -1,4 +1,5 @@
# Copyright 2018 Red Hat, Inc.
# Copyright 2022 OVHCloud
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -12,92 +13,29 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import glance_store as store_api
from glance_store import backend
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import encodeutils
from oslo_utils import excutils
from taskflow.patterns import linear_flow as lf
from taskflow import task
from taskflow.types import failure
from glance.async_.flows._internal_plugins import base_download
from glance.common import exception
from glance.common.scripts import utils as script_utils
from glance.i18n import _, _LE
from glance.i18n import _
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class _WebDownload(task.Task):
default_provides = 'file_uri'
class _WebDownload(base_download.BaseDownload):
def __init__(self, task_id, task_type, uri, action_wrapper, stores):
self.task_id = task_id
self.task_type = task_type
self.image_id = action_wrapper.image_id
self.uri = uri
self.action_wrapper = action_wrapper
self.stores = stores
self._path = None
super(_WebDownload, self).__init__(
name='%s-WebDownload-%s' % (task_type, task_id))
# NOTE(abhishekk): Use reserved 'os_glance_staging_store' for
# staging the data, the else part will be removed once old way
# of configuring store is deprecated.
if CONF.enabled_backends:
self.store = store_api.get_store_from_store_identifier(
'os_glance_staging_store')
else:
if CONF.node_staging_uri is None:
msg = (_("%(task_id)s of %(task_type)s not configured "
"properly. Missing node_staging_uri: %(work_dir)s") %
{'task_id': self.task_id,
'task_type': self.task_type,
'work_dir': CONF.node_staging_uri})
raise exception.BadTaskConfiguration(msg)
self.store = self._build_store()
def _build_store(self):
# NOTE(flaper87): Due to the nice glance_store api (#sarcasm), we're
# forced to build our own config object, register the required options
# (and by required I mean *ALL* of them, even the ones we don't want),
# and create our own store instance by calling a private function.
# This is certainly unfortunate but it's the best we can do until the
# glance_store refactor is done. A good thing is that glance_store is
# under our team's management and it gates on Glance so changes to
# this API will (should?) break task's tests.
# TODO(abhishekk): After removal of backend module from glance_store
# need to change this to use multi_backend module.
conf = cfg.ConfigOpts()
try:
backend.register_opts(conf)
except cfg.DuplicateOptError:
pass
conf.set_override('filesystem_store_datadir',
CONF.node_staging_uri[7:],
group='glance_store')
# NOTE(flaper87): Do not even try to judge me for this... :(
# With the glance_store refactor, this code will change, until
# that happens, we don't have a better option and this is the
# least worst one, IMHO.
store = backend._load_store(conf, 'file')
if store is None:
msg = (_("%(task_id)s of %(task_type)s not configured "
"properly. Could not load the filesystem store") %
{'task_id': self.task_id, 'task_type': self.task_type})
raise exception.BadTaskConfiguration(msg)
store.configure()
return store
super(_WebDownload, self).__init__(task_id, task_type, action_wrapper,
stores, 'WebDownload')
def execute(self):
"""Create temp file into store and return path to it
@ -133,36 +71,6 @@ class _WebDownload(task.Task):
pass
return self._path
def revert(self, result, **kwargs):
if isinstance(result, failure.Failure):
LOG.error(_LE('Task: %(task_id)s failed to import image '
'%(image_id)s to the filesystem.'),
{'task_id': self.task_id,
'image_id': self.image_id})
# NOTE(abhishekk): Revert image state back to 'queued' as
# something went wrong.
# NOTE(danms): If we failed to stage the image, then none
# of the _ImportToStore() tasks could have run, so we need
# to move all stores out of "importing" and into "failed".
with self.action_wrapper as action:
action.set_image_attribute(status='queued')
action.remove_importing_stores(self.stores)
action.add_failed_stores(self.stores)
# NOTE(abhishekk): Deleting partial image data from staging area
if self._path is not None:
LOG.debug(('Deleting image %(image_id)s from staging '
'area.'), {'image_id': self.image_id})
try:
if CONF.enabled_backends:
store_api.delete(self._path, None)
else:
store_api.delete_from_backend(self._path)
except Exception:
LOG.exception(_LE("Error reverting web-download "
"task: %(task_id)s"), {
'task_id': self.task_id})
def get_flow(**kwargs):
"""Return task flow for web-download.

151
glance/async_/flows/api_image_import.py
View File

@ -14,7 +14,9 @@
# under the License.
import copy
import functools
import json
import os
import urllib.request
import glance_store as store_api
from glance_store import backend
@ -33,6 +35,7 @@ from taskflow import task
from glance.api import common as api_common
import glance.async_.flows._internal_plugins as internal_plugins
import glance.async_.flows.plugins as import_plugins
from glance.async_ import utils
from glance.common import exception
from glance.common.scripts.image_import import main as image_import
from glance.common.scripts import utils as script_utils
@ -79,6 +82,30 @@ Possible values:
CONF.register_opts(api_import_opts, group='image_import_opts')
glance_download_opts = [
cfg.ListOpt('extra_properties',
item_type=cfg.types.String(quotes=True),
bounds=True,
default=[
'hw_', 'trait:', 'os_distro', 'os_secure_boot',
'os_type'],
help=_("""
Specify metadata prefix to be set on the target image when using
glance-download. All other properties coming from the source image won't be set
on the target image. If specified metadata does not exist on the source image
it won't be set on the target image. Note you can't set the os_glance prefix
as it is reserved by glance, so the related properties won't be set on the
target image.
Possible values:
* List containing extra_properties prefixes: ['os_', 'architecture']
""")),
]
CONF.register_opts(glance_download_opts, group='glance_download_properties')
# TODO(jokke): We should refactor the task implementations so that we do not
# need to duplicate what we have already for example in base_import.py.
@ -89,6 +116,12 @@ class _NoStoresSucceeded(exception.GlanceException):
super(_NoStoresSucceeded, self).__init__(message)
class _InvalidGlanceDownloadImageStatus(exception.GlanceException):
def __init__(self, message):
super(_InvalidGlanceDownloadImageStatus, self).__init__(message)
class ImportActionWrapper(object):
"""Wrapper for all the image metadata operations we do during an import.
@ -207,6 +240,18 @@ class _ImportActions(object):
# should have moderated access like all the other things here.
return copy.deepcopy(self._image.locations)
@property
def image_disk_format(self):
return self._image.disk_format
@property
def image_container_format(self):
return self._image.container_format
@property
def image_extra_properties(self):
return dict(self._image.extra_properties)
@property
def image_status(self):
return self._image.status
@ -739,6 +784,93 @@ class _CompleteTask(task.Task):
{'task_id': self.task_id, 'task_type': self.task_type})
class _ImportMetadata(task.Task):
default_provides = 'image_size'
def __init__(self, task_id, task_type, context, action_wrapper,
import_req):
self.task_id = task_id
self.task_type = task_type
self.context = context
self.action_wrapper = action_wrapper
self.import_req = import_req
self.props_to_copy = CONF.glance_download_properties.extra_properties
# We store the properties that will be set in case we are reverting
self.properties = {}
self.old_properties = {}
self.old_attributes = {}
super(_ImportMetadata, self).__init__(
name='%s-ImportMetdata-%s' % (task_type, task_id))
def execute(self):
try:
glance_endpoint = utils.get_glance_endpoint(
self.context,
self.import_req['method']['glance_region'],
self.import_req['method']['glance_service_interface'])
glance_image_id = self.import_req['method']['glance_image_id']
image_download_metadata_url = '%s/v2/images/%s' % (
glance_endpoint, glance_image_id)
LOG.info(_LI("Fetching glance image metadata from remote host %s"),
image_download_metadata_url)
token = self.context.auth_token
request = urllib.request.Request(image_download_metadata_url,
headers={'X-Auth-Token': token})
with urllib.request.urlopen(request) as payload:
data = json.loads(payload.read().decode('utf-8'))
if data.get('status') != 'active':
raise _InvalidGlanceDownloadImageStatus(
_('Source image status should be active instead of %s')
% data['status'])
for key, value in data.items():
for metadata in self.props_to_copy:
if key.startswith(metadata):
self.properties[key] = value
with self.action_wrapper as action:
# Save the old properties in case we need to revert
self.old_properties = action.image_extra_properties
self.old_attributes = {
'container_format': action.image_container_format,
'disk_format': action.image_disk_format,
}
# Set disk_format and container_format attributes
action.set_image_attribute(
disk_format=data['disk_format'],
container_format=data['container_format'])
# Set extra propoerties
if self.properties:
action.set_image_extra_properties(self.properties)
try:
return int(data['size'])
except (ValueError, KeyError):
raise exception.ImportTaskError(
_('Size attribute of remote image %s could not be '
'determined.' % glance_image_id))
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.error(
"Task %(task_id)s failed with exception %(error)s", {
"error": encodeutils.exception_to_unicode(e),
"task_id": self.task_id
})
def revert(self, result, **kwargs):
"""Revert the extra properties set and set the image in queued"""
with self.action_wrapper as action:
for image_property in self.properties:
if image_property not in self.old_properties:
action.pop_extra_property(image_property)
action.set_image_extra_properties(self.old_properties)
action.set_image_attribute(status='queued',
**self.old_attributes)
def assert_quota(context, task_repo, task_id, stores,
action_wrapper, enforce_quota_fn,
**enforce_kwargs):
@ -776,11 +908,13 @@ def get_flow(**kwargs):
image_repo = kwargs.get('image_repo')
admin_repo = kwargs.get('admin_repo')
image_id = kwargs.get('image_id')
import_method = kwargs.get('import_req')['method']['name']
uri = kwargs.get('import_req')['method'].get('uri')
import_req = kwargs.get('import_req')
import_method = import_req['method']['name']
uri = import_req['method'].get('uri')
stores = kwargs.get('backend', [None])
all_stores_must_succeed = kwargs.get('import_req').get(
all_stores_must_succeed = import_req.get(
'all_stores_must_succeed', True)
context = kwargs.get('context')
separator = ''
if not CONF.enabled_backends and not CONF.node_staging_uri.endswith('/'):
@ -803,7 +937,10 @@ def get_flow(**kwargs):
flow.add(_ImageLock(task_id, task_type, action_wrapper))
if import_method in ['web-download', 'copy-image']:
if import_method in ['web-download', 'copy-image', 'glance-download']:
if import_method == 'glance-download':
flow.add(_ImportMetadata(task_id, task_type,
context, action_wrapper, import_req))
internal_plugin = internal_plugins.get_import_plugin(**kwargs)
flow.add(internal_plugin)
if CONF.enabled_backends:
@ -874,9 +1011,9 @@ def get_flow(**kwargs):
stores, action_wrapper,
ks_quota.enforce_image_size_total,
delta=image_size)
elif import_method in ('copy-image', 'web-download'):
# The copy-image and web-download methods will use staging space to
# do their work, so check that quota.
elif import_method in ('copy-image', 'web-download', 'glance-download'):
# The copy-image, web-download and glance-download methods will use
# staging space to do their work, so check that quota.
assert_quota(kwargs['context'], task_repo, task_id,
stores, action_wrapper,
ks_quota.enforce_image_staging_total,

2
glance/common/config.py
View File

@ -572,7 +572,7 @@ Related options:
item_type=cfg.types.String(quotes=True),
bounds=True,
default=['glance-direct', 'web-download',
'copy-image'],
'copy-image', 'glance-download'],
help=_("""
List of enabled Image Import Methods

159
glance/tests/unit/async_/flows/test_api_image_import.py
View File

@ -15,6 +15,7 @@
import sys
from unittest import mock
import urllib.error
from glance_store import exceptions as store_exceptions
from oslo_config import cfg
@ -25,6 +26,7 @@ import glance.async_.flows.api_image_import as import_flow
from glance.common import exception
from glance.common.scripts.image_import import main as image_import
from glance import context
from glance.domain import ExtraProperties
from glance import gateway
import glance.tests.utils as test_utils
@ -890,7 +892,9 @@ class TestImportActions(test_utils.BaseTestCase):
self.image = mock.MagicMock()
self.image.image_id = IMAGE_ID1
self.image.status = 'active'
self.image.extra_properties = {'speed': '88mph'}
self.image.disk_format = 'raw'
self.image.container_format = 'bare'
self.image.extra_properties = ExtraProperties({'speed': '88mph'})
self.image.checksum = mock.sentinel.checksum
self.image.os_hash_algo = mock.sentinel.hash_algo
self.image.os_hash_value = mock.sentinel.hash_value
@ -900,6 +904,10 @@ class TestImportActions(test_utils.BaseTestCase):
def test_image_property_proxies(self):
self.assertEqual(IMAGE_ID1, self.actions.image_id)
self.assertEqual('active', self.actions.image_status)
self.assertEqual('raw', self.actions.image_disk_format)
self.assertEqual('bare', self.actions.image_container_format)
self.assertEqual({'speed': '88mph'},
self.actions.image_extra_properties)
def test_merge_store_list(self):
# Addition with no existing property works
@ -1131,3 +1139,152 @@ class TestCompleteTask(test_utils.BaseTestCase):
{'image': IMAGE_ID1,
'task': TASK_ID1})
self.task.succeed.assert_called_once_with({'image_id': IMAGE_ID1})
class TestImportMetadata(test_utils.BaseTestCase):
def setUp(self):
super(TestImportMetadata, self).setUp()
self.config(extra_properties=[],
group="glance_download_properties")
self.wrapper = mock.MagicMock(image_id=IMAGE_ID1)
self.context = context.RequestContext(user_id=TENANT1,
project_id=TENANT1,
overwrite=False)
self.import_req = {
'method': {
'glance_region': 'RegionTwo',
'glance_service_interface': 'public',
'glance_image_id': IMAGE_ID1
}
}
@mock.patch('urllib.request')
@mock.patch('glance.async_.flows.api_image_import.json')
@mock.patch('glance.async_.utils.get_glance_endpoint')
def test_execute_return_image_size(self, mock_gge, mock_json,
mock_request):
self.config(extra_properties=['hw:numa_nodes', 'os_hash'],
group="glance_download_properties")
mock_gge.return_value = 'https://other.cloud.foo/image'
action = self.wrapper.__enter__.return_value
mock_json.loads.return_value = {
'status': 'active',
'disk_format': 'qcow2',
'container_format': 'bare',
'hw:numa_nodes': '2',
'os_hash': 'hash',
'extra_metadata': 'hello',
'size': '12345'
}
task = import_flow._ImportMetadata(TASK_ID1, TASK_TYPE,
self.context, self.wrapper,
self.import_req)
self.assertEqual(12345, task.execute())
mock_request.Request.assert_called_once_with(
'https://other.cloud.foo/image/v2/images/%s' % (
IMAGE_ID1),
headers={'X-Auth-Token': self.context.auth_token})
mock_gge.assert_called_once_with(self.context, 'RegionTwo', 'public')
action.set_image_attribute.assert_called_once_with(
disk_format='qcow2',
container_format='bare')
action.set_image_extra_properties.assert_called_once_with({
'hw:numa_nodes': '2',
'os_hash': 'hash'
})
@mock.patch('urllib.request')
@mock.patch('glance.async_.utils.get_glance_endpoint')
def test_execute_fail_no_glance_endpoint(self, mock_gge, mock_request):
action = self.wrapper.__enter__.return_value
mock_gge.side_effect = exception.GlanceEndpointNotFound(
region='RegionTwo',
interface='public')
task = import_flow._ImportMetadata(TASK_ID1, TASK_TYPE,
self.context, self.wrapper,
self.import_req)
self.assertRaises(exception.GlanceEndpointNotFound,
task.execute)
action.assert_not_called()
mock_request.assert_not_called()
@mock.patch('urllib.request')
@mock.patch('glance.async_.utils.get_glance_endpoint')
def test_execute_fail_remote_glance_unreachable(self, mock_gge, mock_r):
action = self.wrapper.__enter__.return_value
mock_r.urlopen.side_effect = urllib.error.HTTPError(
'/file', 400, 'Test Fail', {}, None)
task = import_flow._ImportMetadata(TASK_ID1, TASK_TYPE,
self.context, self.wrapper,
self.import_req)
self.assertRaises(urllib.error.HTTPError,
task.execute)
action.assert_not_called()
@mock.patch('urllib.request')
@mock.patch('glance.async_.flows.api_image_import.json')
@mock.patch('glance.async_.utils.get_glance_endpoint')
def test_execute_invalid_remote_image_state(self, mock_gge, mock_json,
mock_request):
action = self.wrapper.__enter__.return_value
mock_gge.return_value = 'https://other.cloud.foo/image'
mock_json.loads.return_value = {
'status': 'queued',
}
task = import_flow._ImportMetadata(TASK_ID1, TASK_TYPE,
self.context, self.wrapper,
self.import_req)
self.assertRaises(import_flow._InvalidGlanceDownloadImageStatus,
task.execute)
action.assert_not_called()
@mock.patch('urllib.request')
@mock.patch('glance.async_.flows.api_image_import.json')
@mock.patch('glance.async_.utils.get_glance_endpoint')
def test_execute_raise_if_no_size(self, mock_gge, mock_json, mock_request):
self.config(extra_properties=['hw:numa_nodes', 'os_hash'],
group="glance_download_properties")
mock_gge.return_value = 'https://other.cloud.foo/image'
action = self.wrapper.__enter__.return_value
mock_json.loads.return_value = {
'status': 'active',
'disk_format': 'qcow2',
'container_format': 'bare',
'hw:numa_nodes': '2',
'os_hash': 'hash',
'extra_metadata': 'hello',
}
task = import_flow._ImportMetadata(TASK_ID1, TASK_TYPE,
self.context, self.wrapper,
self.import_req)
self.assertRaises(exception.ImportTaskError, task.execute)
mock_request.Request.assert_called_once_with(
'https://other.cloud.foo/image/v2/images/%s' % (
IMAGE_ID1),
headers={'X-Auth-Token': self.context.auth_token})
mock_gge.assert_called_once_with(self.context, 'RegionTwo', 'public')
action.set_image_attribute.assert_called_once_with(
disk_format='qcow2',
container_format='bare')
action.set_image_extra_properties.assert_called_once_with({
'hw:numa_nodes': '2',
'os_hash': 'hash'
})
def test_revert_rollback_metadata_value(self):
action = self.wrapper.__enter__.return_value
task = import_flow._ImportMetadata(TASK_ID1, TASK_TYPE,
self.context, self.wrapper,
self.import_req)
task.properties = {'prop1': 'value1', 'prop2': 'value2'}
task.old_properties = {'prop1': 'orig_val', 'old_prop': 'old_value'}
task.old_attributes = {'container_format': 'bare',
'disk_format': 'qcow2'}
task.revert(None)
action.set_image_attribute.assert_called_once_with(
status='queued',
container_format='bare',
disk_format='qcow2')
action.pop_extra_property.assert_called_once_with('prop2')
action.set_image_extra_properties.assert_called_once_with(
task.old_properties)

207
glance/tests/unit/async_/flows/test_base_download.py Normal file
View File

@ -0,0 +1,207 @@
# Copyright 2022 OVHCloud
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from unittest import mock
from glance_store import backend
from oslo_config import cfg
from taskflow.types import failure
from glance.async_.flows import api_image_import
import glance.common.exception
from glance import domain
import glance.tests.unit.utils as unit_test_utils
import glance.tests.utils as test_utils
CONF = cfg.CONF
TENANT1 = '6838eb7b-6ded-434a-882c-b344c77fe8df'
class TestBaseDownloadTask(test_utils.BaseTestCase):
def setUp(self):
super(TestBaseDownloadTask, self).setUp()
self.config(node_staging_uri='/tmp/staging')
self.image_repo = mock.MagicMock()
self.image_id = mock.MagicMock()
self.uri = mock.MagicMock()
self.plugin_name = 'FakeBaseDownload'
self.task_factory = domain.TaskFactory()
task_input = {
"import_req": {
'method': {
'name': 'web_download',
'uri': 'http://cloud.foo/image.qcow2'
}
}
}
task_ttl = CONF.task.task_time_to_live
self.task_type = 'import'
request_id = 'fake_request_id'
user_id = 'fake_user'
self.task = self.task_factory.new_task(self.task_type, TENANT1,
self.image_id, user_id,
request_id,
task_time_to_live=task_ttl,
task_input=task_input)
self.task_id = self.task.task_id
self.action_wrapper = api_image_import.ImportActionWrapper(
self.image_repo, self.image_id, self.task_id)
self.image_repo.get.return_value = mock.MagicMock(
extra_properties={'os_glance_import_task': self.task_id})
self.base_download_task = unit_test_utils.FakeBaseDownloadPlugin(
self.task.task_id, self.task_type, self.action_wrapper,
['foo'], self.plugin_name)
self.base_download_task._path = "/path/to_downloaded_data"
def test_base_download_node_staging_uri_is_none(self):
self.config(node_staging_uri=None)
self.assertRaises(glance.common.exception.BadTaskConfiguration,
unit_test_utils.FakeBaseDownloadPlugin,
self.task.task_id, self.task_type, self.uri,
self.action_wrapper, ['foo'])
@mock.patch.object(cfg.ConfigOpts, "set_override")
def test_base_download_node_store_initialization_failed(
self, mock_override):
with mock.patch.object(backend, '_load_store') as mock_load_store:
mock_load_store.return_value = None
self.assertRaises(glance.common.exception.BadTaskConfiguration,
unit_test_utils.FakeBaseDownloadPlugin,
self.task.task_id, self.task_type, self.uri,
self.action_wrapper, ['foo'])
mock_override.assert_called()
def test_base_download_delete_staging_image_not_exist(self):
staging_path = "file:///tmp/staging/temp-image"
delete_from_fs_task = api_image_import._DeleteFromFS(
self.task.task_id, self.task_type)
with mock.patch.object(os.path, "exists") as mock_exists:
mock_exists.return_value = False
with mock.patch.object(os, "unlink") as mock_unlik:
delete_from_fs_task.execute(staging_path)
self.assertEqual(1, mock_exists.call_count)
self.assertEqual(0, mock_unlik.call_count)
@mock.patch.object(os.path, "exists")
def test_base_download_delete_staging_image_failed(self, mock_exists):
mock_exists.return_value = True
staging_path = "file:///tmp/staging/temp-image"
delete_from_fs_task = api_image_import._DeleteFromFS(
self.task.task_id, self.task_type)
with mock.patch.object(os, "unlink") as mock_unlink:
try:
delete_from_fs_task.execute(staging_path)
except OSError:
self.assertEqual(1, mock_unlink.call_count)
self.assertEqual(1, mock_exists.call_count)
@mock.patch.object(os.path, "exists")
def test_base_download_delete_staging_image_succeed(self, mock_exists):
mock_exists.return_value = True
staging_path = "file:///tmp/staging/temp-image"
delete_from_fs_task = api_image_import._DeleteFromFS(
self.task.task_id, self.task_type)
with mock.patch.object(os, "unlink") as mock_unlik:
delete_from_fs_task.execute(staging_path)
self.assertEqual(1, mock_exists.call_count)
self.assertEqual(1, mock_unlik.call_count)
@mock.patch(
"glance.async_.flows._internal_plugins.base_download.store_api")
def test_base_download_revert_with_failure(self, mock_store_api):
image = self.image_repo.get.return_value
image.extra_properties['os_glance_importing_to_stores'] = 'foo'
image.extra_properties['os_glance_failed_import'] = ''
self.base_download_task.execute = mock.MagicMock(
side_effect=glance.common.exception.ImportTaskError)
self.base_download_task.revert(None)
mock_store_api.delete_from_backend.assert_called_once_with(
"/path/to_downloaded_data")
self.assertEqual(1, self.image_repo.save.call_count)
self.assertEqual(
'', image.extra_properties['os_glance_importing_to_stores'])
self.assertEqual(
'foo', image.extra_properties['os_glance_failed_import'])
@mock.patch(
"glance.async_.flows._internal_plugins.base_download.store_api")
def test_base_download_revert_without_failure_multi_store(self,
mock_store_api):
enabled_backends = {
'fast': 'file',
'cheap': 'file'
}
self.config(enabled_backends=enabled_backends)
self.base_download_task.revert("/path/to_downloaded_data")
mock_store_api.delete.assert_called_once_with(
"/path/to_downloaded_data", None)
@mock.patch(
"glance.async_.flows._internal_plugins.base_download.store_api")
def test_base_download_revert_with_failure_without_path(self,
mock_store_api):
image = self.image_repo.get.return_value
image.status = 'importing'
image.extra_properties['os_glance_importing_to_stores'] = 'foo'
image.extra_properties['os_glance_failed_import'] = ''
result = failure.Failure.from_exception(
glance.common.exception.ImportTaskError())
self.base_download_task._path = None
self.base_download_task.revert(result)
mock_store_api.delete_from_backend.assert_not_called()
# NOTE(danms): Since we told revert that we were the problem,
# we should have updated the image status and moved the stores
# to the failed list.
self.image_repo.save.assert_called_once_with(image, 'importing')
self.assertEqual('queued', image.status)
self.assertEqual(
'', image.extra_properties['os_glance_importing_to_stores'])
self.assertEqual(
'foo', image.extra_properties['os_glance_failed_import'])
@mock.patch(
"glance.async_.flows._internal_plugins.base_download.store_api")
def test_base_download_revert_with_failure_with_path(self, mock_store_api):
result = failure.Failure.from_exception(
glance.common.exception.ImportTaskError())
self.base_download_task.revert(result)
mock_store_api.delete_from_backend.assert_called_once_with(
"/path/to_downloaded_data")
@mock.patch(
"glance.async_.flows._internal_plugins.base_download.store_api")
def test_base_download_delete_fails_on_revert(self, mock_store_api):
result = failure.Failure.from_exception(
glance.common.exception.ImportTaskError())
mock_store_api.delete_from_backend.side_effect = Exception
# this will verify that revert does not break because of failure
# while deleting data in staging area
self.base_download_task.revert(result)

167
glance/tests/unit/async_/flows/test_glance_download.py Normal file
View File

@ -0,0 +1,167 @@
# Copyright 2022 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from unittest import mock
import urllib.error
from glance_store._drivers import filesystem
from oslo_config import cfg
from oslo_utils.fixture import uuidsentinel
from glance.async_.flows._internal_plugins import glance_download
from glance.async_.flows import api_image_import
import glance.common.exception
import glance.context
from glance import domain
import glance.tests.utils as test_utils
CONF = cfg.CONF
TENANT1 = '6838eb7b-6ded-434a-882c-b344c77fe8df'
class TestGlanceDownloadTask(test_utils.BaseTestCase):
def setUp(self):
super(TestGlanceDownloadTask, self).setUp()
self.config(node_staging_uri='/tmp/staging')
self.image_repo = mock.MagicMock()
self.image_id = mock.MagicMock()
self.uri = mock.MagicMock()
self.task_factory = domain.TaskFactory()
self.context = glance.context.RequestContext(tenant=TENANT1,
auth_token='token')
task_input = {
"import_req": {
'method': {
'name': 'glance-download',
'glance_image_id': uuidsentinel.remote_image,
'glance_region': 'RegionTwo',
'glance_service_interface': 'public',
}
}
}
task_ttl = CONF.task.task_time_to_live
self.task_type = 'import'
request_id = 'fake_request_id'
user_id = 'fake_user'
self.task = self.task_factory.new_task(self.task_type, TENANT1,
self.image_id, user_id,
request_id,
task_time_to_live=task_ttl,
task_input=task_input)
self.task_id = self.task.task_id
self.action_wrapper = api_image_import.ImportActionWrapper(
self.image_repo, self.image_id, self.task_id)
self.image_repo.get.return_value = mock.MagicMock(
extra_properties={'os_glance_import_task': self.task_id})
@mock.patch.object(filesystem.Store, 'add')
@mock.patch('glance.async_.utils.get_glance_endpoint')
def test_glance_download(self, mock_gge, mock_add):
mock_gge.return_value = 'https://other.cloud.foo/image'
glance_download_task = glance_download._DownloadGlanceImage(
self.context, self.task.task_id, self.task_type,
self.action_wrapper, ['foo'],
'RegionTwo', uuidsentinel.remote_image, 'public')
with mock.patch('urllib.request') as mock_request:
mock_add.return_value = ["path", 12345]
self.assertEqual(glance_download_task.execute(12345), "path")
mock_add.assert_called_once_with(
self.image_id,
mock_request.urlopen.return_value, 0)
mock_request.Request.assert_called_once_with(
'https://other.cloud.foo/image/v2/images/%s/file' % (
uuidsentinel.remote_image),
headers={'X-Auth-Token': self.context.auth_token})
mock_gge.assert_called_once_with(self.context, 'RegionTwo', 'public')
@mock.patch.object(filesystem.Store, 'add')
@mock.patch('glance.async_.utils.get_glance_endpoint')
def test_glance_download_failed(self, mock_gge, mock_add):
mock_gge.return_value = 'https://other.cloud.foo/image'
glance_download_task = glance_download._DownloadGlanceImage(
self.context, self.task.task_id, self.task_type,
self.action_wrapper, ['foo'],
'RegionTwo', uuidsentinel.remote_image, 'public')
with mock.patch('urllib.request') as mock_request:
mock_request.urlopen.side_effect = urllib.error.HTTPError(
'/file', 400, 'Test Fail', {}, None)
self.assertRaises(urllib.error.HTTPError,
glance_download_task.execute,
12345)
mock_add.assert_not_called()
mock_request.Request.assert_called_once_with(
'https://other.cloud.foo/image/v2/images/%s/file' % (
uuidsentinel.remote_image),
headers={'X-Auth-Token': self.context.auth_token})
mock_gge.assert_called_once_with(self.context, 'RegionTwo', 'public')
@mock.patch('urllib.request')
@mock.patch('glance.async_.utils.get_glance_endpoint')
def test_glance_download_no_glance_endpoint(self, mock_gge, mock_request):
mock_gge.side_effect = glance.common.exception.GlanceEndpointNotFound(
region='RegionTwo',
interface='public')
glance_download_task = glance_download._DownloadGlanceImage(
self.context, self.task.task_id, self.task_type,
self.action_wrapper, ['foo'],
'RegionTwo', uuidsentinel.remote_image, 'public')
self.assertRaises(glance.common.exception.GlanceEndpointNotFound,
glance_download_task.execute, 12345)
mock_request.assert_not_called()
@mock.patch.object(filesystem.Store, 'add')
@mock.patch('glance.async_.utils.get_glance_endpoint')
def test_glance_download_size_mismatch(self, mock_gge, mock_add):
mock_gge.return_value = 'https://other.cloud.foo/image'
glance_download_task = glance_download._DownloadGlanceImage(
self.context, self.task.task_id, self.task_type,
self.action_wrapper, ['foo'],
'RegionTwo', uuidsentinel.remote_image, 'public')
with mock.patch('urllib.request') as mock_request:
mock_add.return_value = ["path", 1]
self.assertRaises(glance.common.exception.ImportTaskError,
glance_download_task.execute, 12345)
mock_add.assert_called_once_with(
self.image_id,
mock_request.urlopen.return_value, 0)
mock_request.Request.assert_called_once_with(
'https://other.cloud.foo/image/v2/images/%s/file' % (
uuidsentinel.remote_image),
headers={'X-Auth-Token': self.context.auth_token})
mock_gge.assert_called_once_with(self.context, 'RegionTwo', 'public')
@mock.patch('urllib.request')
@mock.patch('glance.common.utils.validate_import_uri')
@mock.patch('glance.async_.utils.get_glance_endpoint')
def test_glance_download_wrong_download_url(self, mock_gge, mock_validate,
mock_request):
mock_validate.return_value = False
mock_gge.return_value = 'https://other.cloud.foo/image'
glance_download_task = glance_download._DownloadGlanceImage(
self.context, self.task.task_id, self.task_type,
self.action_wrapper, ['foo'],
'RegionTwo', uuidsentinel.remote_image, 'public')
self.assertRaises(glance.common.exception.ImportTaskError,
glance_download_task.execute, 12345)
mock_request.assert_not_called()
mock_validate.assert_called_once_with(
'https://other.cloud.foo/image/v2/images/%s/file' % (
uuidsentinel.remote_image))

196
glance/tests/unit/async_/flows/test_web_download.py
View File

@ -13,13 +13,10 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
from unittest import mock
from glance_store._drivers import filesystem
from glance_store import backend
from oslo_config import cfg
from taskflow.types import failure
from glance.async_.flows._internal_plugins import web_download
from glance.async_.flows import api_image_import
@ -67,217 +64,84 @@ class TestWebDownloadTask(test_utils.BaseTestCase):
self.task_id = self.task.task_id
self.action_wrapper = api_image_import.ImportActionWrapper(
self.image_repo, self.image_id, self.task_id)
self.web_download_task = web_download._WebDownload(
self.task.task_id, self.task_type, self.uri, self.action_wrapper,
['foo'])
self.image_repo.get.return_value = mock.MagicMock(
extra_properties={'os_glance_import_task': self.task_id})
@mock.patch.object(filesystem.Store, 'add')
def test_web_download(self, mock_add):
web_download_task = web_download._WebDownload(
self.task.task_id, self.task_type, self.uri, self.action_wrapper,
['foo'])
with mock.patch.object(script_utils,
'get_image_data_iter') as mock_iter:
mock_add.return_value = ["path", 4]
mock_iter.return_value.headers = {}
self.assertEqual(web_download_task.execute(), "path")
self.assertEqual(self.web_download_task.execute(), "path")
mock_add.assert_called_once_with(self.image_id,
mock_iter.return_value, 0)
@mock.patch.object(filesystem.Store, 'add')
def test_web_download_with_content_length(self, mock_add):
web_download_task = web_download._WebDownload(
self.task.task_id, self.task_type, self.uri, self.action_wrapper,
['foo'])
with mock.patch.object(script_utils,
'get_image_data_iter') as mock_iter:
mock_iter.return_value.headers = {'content-length': '4'}
mock_add.return_value = ["path", 4]
self.assertEqual(web_download_task.execute(), "path")
self.assertEqual(self.web_download_task.execute(), "path")
mock_add.assert_called_once_with(self.image_id,
mock_iter.return_value, 0)
@mock.patch.object(filesystem.Store, 'add')
def test_web_download_with_invalid_content_length(self, mock_add):
web_download_task = web_download._WebDownload(
self.task.task_id, self.task_type, self.uri, self.action_wrapper,
['foo'])
with mock.patch.object(script_utils,
'get_image_data_iter') as mock_iter:
mock_iter.return_value.headers = {'content-length': "not_valid"}
mock_add.return_value = ["path", 4]
self.assertEqual(web_download_task.execute(), "path")
self.assertEqual(self.web_download_task.execute(), "path")
mock_add.assert_called_once_with(self.image_id,
mock_iter.return_value, 0)
@mock.patch.object(filesystem.Store, 'add')
def test_web_download_fails_when_data_size_different(self, mock_add):
web_download_task = web_download._WebDownload(
self.task.task_id, self.task_type, self.uri, self.action_wrapper,
['foo'])
with mock.patch.object(script_utils,
'get_image_data_iter') as mock_iter:
mock_iter.return_value.headers = {'content-length': '4'}
mock_add.return_value = ["path", 3]
self.assertRaises(
glance.common.exception.ImportTaskError,
web_download_task.execute)
def test_web_download_node_staging_uri_is_none(self):
self.config(node_staging_uri=None)
self.assertRaises(glance.common.exception.BadTaskConfiguration,
web_download._WebDownload, self.task.task_id,
self.task_type, self.uri, self.action_wrapper,
['foo'])
@mock.patch.object(cfg.ConfigOpts, "set_override")
def test_web_download_node_store_initialization_failed(self,
mock_override):
with mock.patch.object(backend, '_load_store') as mock_load_store:
mock_load_store.return_value = None
self.assertRaises(glance.common.exception.BadTaskConfiguration,
web_download._WebDownload, self.task.task_id,
self.task_type, self.uri, self.action_wrapper,
['foo'])
mock_override.assert_called()
self.web_download_task.execute)
def test_web_download_failed(self):
web_download_task = web_download._WebDownload(
self.task.task_id, self.task_type, self.uri, self.action_wrapper,
['foo'])
with mock.patch.object(script_utils,
"get_image_data_iter") as mock_iter:
mock_iter.side_effect = glance.common.exception.NotFound
self.assertRaises(glance.common.exception.NotFound,
web_download_task.execute)
def test_web_download_delete_staging_image_not_exist(self):
staging_path = "file:///tmp/staging/temp-image"
delete_from_fs_task = api_image_import._DeleteFromFS(
self.task.task_id, self.task_type)
with mock.patch.object(os.path, "exists") as mock_exists:
mock_exists.return_value = False
with mock.patch.object(os, "unlink") as mock_unlik:
delete_from_fs_task.execute(staging_path)
self.assertEqual(1, mock_exists.call_count)
self.assertEqual(0, mock_unlik.call_count)
@mock.patch.object(os.path, "exists")
def test_web_download_delete_staging_image_failed(self, mock_exists):
mock_exists.return_value = True
staging_path = "file:///tmp/staging/temp-image"
delete_from_fs_task = api_image_import._DeleteFromFS(
self.task.task_id, self.task_type)
with mock.patch.object(os, "unlink") as mock_unlink:
try:
delete_from_fs_task.execute(staging_path)
except OSError:
self.assertEqual(1, mock_unlink.call_count)
self.assertEqual(1, mock_exists.call_count)
@mock.patch.object(os.path, "exists")
def test_web_download_delete_staging_image_succeed(self, mock_exists):
mock_exists.return_value = True
staging_path = "file:///tmp/staging/temp-image"
delete_from_fs_task = api_image_import._DeleteFromFS(
self.task.task_id, self.task_type)
with mock.patch.object(os, "unlink") as mock_unlik:
delete_from_fs_task.execute(staging_path)
self.assertEqual(1, mock_exists.call_count)
self.assertEqual(1, mock_unlik.call_count)
self.web_download_task.execute)
@mock.patch.object(filesystem.Store, 'add')
@mock.patch("glance.async_.flows._internal_plugins.web_download.store_api")
def test_web_download_revert_with_failure(self, mock_store_api,
mock_add):
image = self.image_repo.get.return_value
image.extra_properties['os_glance_importing_to_stores'] = 'foo'
image.extra_properties['os_glance_failed_import'] = ''
web_download_task = web_download._WebDownload(
self.task.task_id, self.task_type, self.uri, self.action_wrapper,
['foo'])
def test_web_download_check_content_length(self, mock_add):
with mock.patch.object(script_utils,
'get_image_data_iter') as mock_iter:
mock_add.return_value = ["path", 4]
mock_iter.return_value.headers = {'content-length': '4'}
mock_add.return_value = "/path/to_downloaded_data", 3
self.assertRaises(
glance.common.exception.ImportTaskError,
web_download_task.execute)
self.assertEqual(self.web_download_task.execute(), "path")
mock_add.assert_called_once_with(self.image_id,
mock_iter.return_value, 0)
web_download_task.revert(None)
mock_store_api.delete_from_backend.assert_called_once_with(
"/path/to_downloaded_data")
# NOTE(danms): Since we told revert that we were not at fault,
# we should not have updated the image.
self.image_repo.save.assert_not_called()
self.assertEqual(
'foo', image.extra_properties['os_glance_importing_to_stores'])
self.assertEqual(
'', image.extra_properties['os_glance_failed_import'])
@mock.patch.object(filesystem.Store, 'add')
def test_web_download_invalid_content_length(self, mock_add):
with mock.patch.object(script_utils,
'get_image_data_iter') as mock_iter:
mock_add.return_value = ["path", 4]
mock_iter.return_value.headers = {'content-length': 'not_valid'}
self.assertEqual(self.web_download_task.execute(), "path")
mock_add.assert_called_once_with(self.image_id,
mock_iter.return_value, 0)
@mock.patch("glance.async_.flows._internal_plugins.web_download.store_api")
def test_web_download_revert_without_failure_multi_store(self,
mock_store_api):
enabled_backends = {
'fast': 'file',
'cheap': 'file'
}
self.config(enabled_backends=enabled_backends)
web_download_task = web_download._WebDownload(
self.task.task_id, self.task_type, self.uri, self.action_wrapper,
['foo'])
web_download_task._path = "/path/to_downloaded_data"
web_download_task.revert("/path/to_downloaded_data")
mock_store_api.delete.assert_called_once_with(
"/path/to_downloaded_data", None)
@mock.patch("glance.async_.flows._internal_plugins.web_download.store_api")
def test_web_download_revert_with_failure_without_path(self,
mock_store_api):
image = self.image_repo.get.return_value
image.status = 'importing'
image.extra_properties['os_glance_importing_to_stores'] = 'foo'
image.extra_properties['os_glance_failed_import'] = ''
result = failure.Failure.from_exception(
glance.common.exception.ImportTaskError())
web_download_task = web_download._WebDownload(
self.task.task_id, self.task_type, self.uri, self.action_wrapper,
['foo'])
web_download_task.revert(result)
mock_store_api.delete_from_backend.assert_not_called()
# NOTE(danms): Since we told revert that we were the problem,
# we should have updated the image status and moved the stores
# to the failed list.
self.image_repo.save.assert_called_once_with(image, 'importing')
self.assertEqual('queued', image.status)
self.assertEqual(
'', image.extra_properties['os_glance_importing_to_stores'])
self.assertEqual(
'foo', image.extra_properties['os_glance_failed_import'])
@mock.patch("glance.async_.flows._internal_plugins.web_download.store_api")
def test_web_download_revert_with_failure_with_path(self, mock_store_api):
result = failure.Failure.from_exception(
glance.common.exception.ImportTaskError())
web_download_task = web_download._WebDownload(
self.task.task_id, self.task_type, self.uri, self.action_wrapper,
['foo'])
web_download_task._path = "/path/to_downloaded_data"
web_download_task.revert(result)
mock_store_api.delete_from_backend.assert_called_once_with(
"/path/to_downloaded_data")
@mock.patch("glance.async_.flows._internal_plugins.web_download.store_api")
def test_web_download_delete_fails_on_revert(self, mock_store_api):
result = failure.Failure.from_exception(
glance.common.exception.ImportTaskError())
mock_store_api.delete_from_backend.side_effect = Exception
web_download_task = web_download._WebDownload(
self.task.task_id, self.task_type, self.uri, self.action_wrapper,
['foo'])
web_download_task._path = "/path/to_downloaded_data"
# this will verify that revert does not break because of failure
# while deleting data in staging area
web_download_task.revert(result)
@mock.patch.object(filesystem.Store, 'add')
def test_web_download_wrong_content_length(self, mock_add):
with mock.patch.object(script_utils,
'get_image_data_iter') as mock_iter:
mock_add.return_value = ["path", 2]
mock_iter.return_value.headers = {'content-length': '4'}
self.assertRaises(glance.common.exception.ImportTaskError,
self.web_download_task.execute)

6
glance/tests/unit/utils.py
View File

@ -21,6 +21,7 @@ import urllib
from oslo_config import cfg
from glance.async_.flows._internal_plugins import base_download
from glance.common import exception
from glance.common import store_utils
from glance.common import wsgi
@ -373,3 +374,8 @@ class FakeTask(object):
def fail(self, message):
self.message = message
self._status = 'failure'
class FakeBaseDownloadPlugin(base_download.BaseDownload):
def execute(self):
pass

2
glance/tests/unit/v2/test_discovery_image_import.py
View File

@ -36,7 +36,7 @@ class TestInfoControllers(test_utils.BaseTestCase):
# TODO(rosmaita): change this when import methods are
# listed in the config file
import_methods = ['glance-direct', 'web-download',
'copy-image']
'copy-image', 'glance-download']
req = unit_test_utils.get_fake_request()
output = self.controller.get_image_import(req)

41
glance/tests/unit/v2/test_images_resource.py
View File

@ -911,6 +911,45 @@ class TestImagesController(base.IsolatedUnitTest):
{'method': {'name': 'web-download',
'uri': 'fake_uri'}})
def test_image_import_raises_bad_request_for_glance_download_missing_input(
self):
request = unit_test_utils.get_fake_request()
with mock.patch.object(
glance.notifier.ImageRepoProxy, 'get') as mock_get:
mock_get.return_value = FakeImage(status='queued')
self.assertRaises(webob.exc.HTTPBadRequest,
self.controller.import_image, request, UUID4,
{'method': {'name': 'glance-download'}})
def test_image_import_raise_bad_request_wrong_id_for_glance_download(
self):
request = unit_test_utils.get_fake_request()
with mock.patch.object(
glance.notifier.ImageRepoProxy, 'get') as mock_get:
mock_get.return_value = FakeImage(status='queued')
self.assertRaises(webob.exc.HTTPBadRequest,
self.controller.import_image, request, UUID4,
{'method': {'name': 'glance-download',
'glance_image_id': 'fake_id',
'glance_region': 'REGION4'}})
@mock.patch.object(glance.domain.TaskFactory, 'new_task')
@mock.patch.object(glance.notifier.ImageRepoProxy, 'get')
def test_image_import_add_default_service_endpoint_for_glance_download(
self, mock_get, mock_nt):
request = unit_test_utils.get_fake_request()
mock_get.return_value = FakeImage(status='queued')
body = {'method': {'name': 'glance-download',
'glance_image_id': UUID4,
'glance_region': 'REGION2'}}
self.controller.import_image(request, UUID4, body)
expected_req = {'method': {'name': 'glance-download',
'glance_image_id': UUID4,
'glance_region': 'REGION2',
'glance_service_interface': 'public'}}
self.assertEqual(expected_req,
mock_nt.call_args.kwargs['task_input']['import_req'])
@mock.patch('glance.context.get_ksa_client')
def test_image_import_proxies(self, mock_client):
# Make sure that we proxy to the remote side when we need to
@ -4865,7 +4904,7 @@ class TestImagesDeserializer(test_utils.BaseTestCase):
request.body = jsonutils.dump_as_bytes(import_body)
return request
KNOWN_IMPORT_METHODS = ['glance-direct', 'web-download']
KNOWN_IMPORT_METHODS = ['glance-direct', 'web-download', 'glance-download']
def test_import_image_invalid_import_method(self):
# Bug 1754634: make sure that what's considered valid

10
releasenotes/notes/add-glance-download-method-be6d9e927b8b0a43.yaml Normal file
View File

@ -0,0 +1,10 @@
---
features:
- |
Glance to glance image import plugin. With this release users can import
an image from an other glance server from an other opensatck region.
The two glance services must use the same keystone service. The feature
use the same keystone authentication token on both glance services and
copy by default container_format, disk_format and customizable properties
from source image ``['hw_', 'trait:', 'os_distro', 'os_secure_boot',
'os_type']``

1
setup.cfg
View File

@ -82,6 +82,7 @@ glance.image_import.plugins =
glance.image_import.internal_plugins =
web_download = glance.async_.flows._internal_plugins.web_download:get_flow
copy_image = glance.async_.flows._internal_plugins.copy_image:get_flow
glance_download = glance.async_.flows._internal_plugins.glance_download:get_flow
[egg_info]