diff --git a/glance/api/v2/discovery.py b/glance/api/v2/discovery.py index 563ac08aa3..10befde80c 100644 --- a/glance/api/v2/discovery.py +++ b/glance/api/v2/discovery.py @@ -32,13 +32,10 @@ class InfoController(object): raise webob.exc.HTTPNotFound(explanation=msg) # TODO(jokke): All the rest of the boundaries should be implemented. - # TODO(jokke): Once we have the rest of the methods implemented - # the value should be inherited from the CONF rather than hard- - # coded. import_methods = { 'description': 'Import methods available.', 'type': 'array', - 'value': ['glance-direct'] + 'value': CONF.get('enabled_import_methods') } return { diff --git a/glance/api/v2/images.py b/glance/api/v2/images.py index a07780c1e4..6f8f71c7c0 100644 --- a/glance/api/v2/images.py +++ b/glance/api/v2/images.py @@ -94,7 +94,8 @@ class ImagesController(object): executor_factory = self.gateway.get_task_executor_factory(req.context) task_repo = self.gateway.get_task_repo(req.context) - task_input = {'image_id': image_id} + task_input = {'image_id': image_id, + 'import_req': body} try: import_task = task_factory.new_task(task_type='api_image_import', @@ -799,7 +800,7 @@ class RequestDeserializer(wsgi.JSONRequestDeserializer): except KeyError: msg = _("Import request requires a 'name' field.") raise webob.exc.HTTPBadRequest(explanation=msg) - if method_name != 'glance-direct': + if method_name not in ['glance-direct', 'web-download']: msg = _("Unknown import method name '%s'.") % method_name raise webob.exc.HTTPBadRequest(explanation=msg) diff --git a/glance/async/flows/_internal_plugins/__init__.py b/glance/async/flows/_internal_plugins/__init__.py new file mode 100644 index 0000000000..3324ae5d7d --- /dev/null +++ b/glance/async/flows/_internal_plugins/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2018 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from stevedore import named + + +CONF = cfg.CONF + + +def get_import_plugin(**kwargs): + method_list = CONF.enabled_import_methods + import_method = kwargs.get('import_req')['method']['name'].replace("-", + "_") + if import_method in method_list: + task_list = [import_method] + # TODO(jokke): Implement error handling of non-listed methods. + extensions = named.NamedExtensionManager( + 'glance.image_import.internal_plugins', + names=task_list, + name_order=True, + invoke_on_load=True, + invoke_kwds=kwargs) + for extension in extensions.extensions: + return extension.obj diff --git a/glance/async/flows/_internal_plugins/web_download.py b/glance/async/flows/_internal_plugins/web_download.py new file mode 100644 index 0000000000..d83e5020b2 --- /dev/null +++ b/glance/async/flows/_internal_plugins/web_download.py @@ -0,0 +1,127 @@ +# Copyright 2018 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from glance_store import backend +from oslo_config import cfg +from oslo_log import log as logging +from taskflow.patterns import linear_flow as lf +from taskflow import task +from taskflow.types import failure + +from glance.common import exception +from glance.common.scripts import utils as script_utils +from glance.i18n import _, _LE + +LOG = logging.getLogger(__name__) + +CONF = cfg.CONF + + +class _WebDownload(task.Task): + + default_provides = 'file_uri' + + def __init__(self, task_id, task_type, task_repo, image_id, uri): + self.task_id = task_id + self.task_type = task_type + self.task_repo = task_repo + self.image_id = image_id + self.uri = uri + super(_WebDownload, self).__init__( + name='%s-WebDownload-%s' % (task_type, task_id)) + + if CONF.node_staging_uri is None: + msg = (_("%(task_id)s of %(task_type)s not configured " + "properly. Missing node_staging_uri: %(work_dir)s") % + {'task_id': self.task_id, + 'task_type': self.task_type, + 'work_dir': CONF.node_staging_uri}) + raise exception.BadTaskConfiguration(msg) + + self.store = self._build_store() + + def _build_store(self): + # NOTE(flaper87): Due to the nice glance_store api (#sarcasm), we're + # forced to build our own config object, register the required options + # (and by required I mean *ALL* of them, even the ones we don't want), + # and create our own store instance by calling a private function. + # This is certainly unfortunate but it's the best we can do until the + # glance_store refactor is done. A good thing is that glance_store is + # under our team's management and it gates on Glance so changes to + # this API will (should?) break task's tests. + conf = cfg.ConfigOpts() + backend.register_opts(conf) + conf.set_override('filesystem_store_datadir', + CONF.node_staging_uri[7:], + group='glance_store') + + # NOTE(flaper87): Do not even try to judge me for this... :( + # With the glance_store refactor, this code will change, until + # that happens, we don't have a better option and this is the + # least worst one, IMHO. + store = backend._load_store(conf, 'file') + + if store is None: + msg = (_("%(task_id)s of %(task_type)s not configured " + "properly. Could not load the filesystem store") % + {'task_id': self.task_id, 'task_type': self.task_type}) + raise exception.BadTaskConfiguration(msg) + + store.configure() + return store + + def execute(self): + """Create temp file into store and return path to it + + :param image_id: Glance Image ID + """ + # NOTE(jokke): We've decided to use staging area for this task as + # a way to expect users to configure a local store for pre-import + # works on the image to happen. + # + # While using any path should be "technically" fine, it's not what + # we recommend as the best solution. For more details on this, please + # refer to the comment in the `_ImportToStore.execute` method. + data = script_utils.get_image_data_iter(self.uri) + + path = self.store.add(self.image_id, data, 0)[0] + + return path + + def revert(self, result, **kwargs): + if isinstance(result, failure.Failure): + LOG.exception(_LE('Task: %(task_id)s failed to import image ' + '%(image_id)s to the filesystem.'), + {'task_id': self.task_id, + 'image_id': self.image_id}) + + +def get_flow(**kwargs): + """Return task flow for web-download. + + :param task_id: Task ID. + :param task_type: Type of the task. + :param image_repo: Image repository used. + :param uri: URI the image data is downloaded from. + """ + task_id = kwargs.get('task_id') + task_type = kwargs.get('task_type') + image_repo = kwargs.get('image_repo') + image_id = kwargs.get('image_id') + uri = kwargs.get('import_req')['method'].get('uri') + + return lf.Flow(task_type).add( + _WebDownload(task_id, task_type, image_repo, image_id, uri), + ) diff --git a/glance/async/flows/api_image_import.py b/glance/async/flows/api_image_import.py index 591898da70..cdab9b3d30 100644 --- a/glance/async/flows/api_image_import.py +++ b/glance/async/flows/api_image_import.py @@ -23,6 +23,7 @@ from taskflow.patterns import linear_flow as lf from taskflow import retry from taskflow import task +import glance.async.flows._internal_plugins as internal_plugins import glance.async.flows.plugins as import_plugins from glance.common import exception from glance.common.scripts.image_import import main as image_import @@ -359,16 +360,23 @@ def get_flow(**kwargs): task_repo = kwargs.get('task_repo') image_repo = kwargs.get('image_repo') image_id = kwargs.get('image_id') - uri = kwargs.get('uri') + import_method = kwargs.get('import_req')['method']['name'] + uri = kwargs.get('import_req')['method'].get('uri') - if not uri: + if not uri and import_method == 'glance-direct': separator = '' if not CONF.node_staging_uri.endswith('/'): separator = '/' uri = separator.join((CONF.node_staging_uri, str(image_id))) flow = lf.Flow(task_type, retry=retry.AlwaysRevert()) - flow.add(_VerifyStaging(task_id, task_type, task_repo, uri)) + + if uri.startswith("http") and import_method == 'web-download': + downloadToStaging = internal_plugins.get_import_plugin(**kwargs) + flow.add(downloadToStaging) + else: + file_uri = uri + flow.add(_VerifyStaging(task_id, task_type, task_repo, file_uri)) for plugin in import_plugins.get_import_plugins(**kwargs): flow.add(plugin) @@ -376,7 +384,7 @@ def get_flow(**kwargs): import_to_store = _ImportToStore(task_id, task_type, image_repo, - uri, + file_uri, image_id) flow.add(import_to_store) diff --git a/glance/common/config.py b/glance/common/config.py index 24dbe19d62..20812a48c7 100644 --- a/glance/common/config.py +++ b/glance/common/config.py @@ -726,6 +726,18 @@ to Image Import Refactoring work. Related options: * [DEFAULT]/node_staging_uri""")), + cfg.ListOpt('enabled_import_methods', + item_type=cfg.types.String(quotes=True), + bounds=True, + default=['glance-direct', 'web-download'], + help=_(""" +List of enabled Image Import Methods + +Both 'glance-direct' and 'web-download' are enabled by default. + +Related options: + * [DEFAULT]/node_staging_uri + * [DEFAULT]/enable_image_import""")), ] CONF = cfg.CONF diff --git a/glance/tests/unit/v2/test_discovery_image_import.py b/glance/tests/unit/v2/test_discovery_image_import.py index 43667b1b84..cce0d3b880 100644 --- a/glance/tests/unit/v2/test_discovery_image_import.py +++ b/glance/tests/unit/v2/test_discovery_image_import.py @@ -37,10 +37,10 @@ class TestInfoControllers(test_utils.BaseTestCase): def test_get_import_info(self): # TODO(rosmaita): change this when import methods are # listed in the config file - import_method = 'glance-direct' + import_methods = ['glance-direct', 'web-download'] self.config(enable_image_import=True) req = unit_test_utils.get_fake_request() output = self.controller.get_image_import(req) self.assertIn('import-methods', output) - self.assertEqual([import_method], output['import-methods']['value']) + self.assertEqual(import_methods, output['import-methods']['value']) diff --git a/setup.cfg b/setup.cfg index a791920d77..00b3bbe45d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,6 +76,9 @@ glance.image_import.plugins = no_op = glance.async.flows.plugins.no_op:get_flow inject_image_metadata=glance.async.flows.plugins.inject_image_metadata:get_flow +glance.image_import.internal_plugins = + web_download = glance.async.flows._internal_plugins.web_download:get_flow + [build_sphinx] builder = html man all_files = 1