Merge "Adds 'web-download' import method"

glance/api/v2/discovery.py

@@ -32,13 +32,10 @@ class InfoController(object):
             raise webob.exc.HTTPNotFound(explanation=msg)
 
         # TODO(jokke): All the rest of the boundaries should be implemented.
-        # TODO(jokke): Once we have the rest of the methods implemented
-        # the value should be inherited from the CONF rather than hard-
-        # coded.
         import_methods = {
             'description': 'Import methods available.',
             'type': 'array',
-            'value': ['glance-direct']
+            'value': CONF.get('enabled_import_methods')
         }
 
         return {

glance/api/v2/images.py

@@ -94,7 +94,8 @@ class ImagesController(object):
         executor_factory = self.gateway.get_task_executor_factory(req.context)
         task_repo = self.gateway.get_task_repo(req.context)
 
-        task_input = {'image_id': image_id}
+        task_input = {'image_id': image_id,
+                      'import_req': body}
 
         try:
             import_task = task_factory.new_task(task_type='api_image_import',
@@ -799,7 +800,7 @@ class RequestDeserializer(wsgi.JSONRequestDeserializer):
         except KeyError:
             msg = _("Import request requires a 'name' field.")
             raise webob.exc.HTTPBadRequest(explanation=msg)
-        if method_name != 'glance-direct':
+        if method_name not in ['glance-direct', 'web-download']:
             msg = _("Unknown import method name '%s'.") % method_name
             raise webob.exc.HTTPBadRequest(explanation=msg)
 

glance/async/flows/_internal_plugins/__init__.py

@@ -0,0 +1,37 @@
+# Copyright 2018 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from stevedore import named
+
+
+CONF = cfg.CONF
+
+
+def get_import_plugin(**kwargs):
+    method_list = CONF.enabled_import_methods
+    import_method = kwargs.get('import_req')['method']['name'].replace("-",
+                                                                       "_")
+    if import_method in method_list:
+        task_list = [import_method]
+        # TODO(jokke): Implement error handling of non-listed methods.
+    extensions = named.NamedExtensionManager(
+        'glance.image_import.internal_plugins',
+        names=task_list,
+        name_order=True,
+        invoke_on_load=True,
+        invoke_kwds=kwargs)
+    for extension in extensions.extensions:
+        return extension.obj

glance/async/flows/_internal_plugins/web_download.py

@@ -0,0 +1,127 @@
+# Copyright 2018 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from glance_store import backend
+from oslo_config import cfg
+from oslo_log import log as logging
+from taskflow.patterns import linear_flow as lf
+from taskflow import task
+from taskflow.types import failure
+
+from glance.common import exception
+from glance.common.scripts import utils as script_utils
+from glance.i18n import _, _LE
+
+LOG = logging.getLogger(__name__)
+
+CONF = cfg.CONF
+
+
+class _WebDownload(task.Task):
+
+    default_provides = 'file_uri'
+
+    def __init__(self, task_id, task_type, task_repo, image_id, uri):
+        self.task_id = task_id
+        self.task_type = task_type
+        self.task_repo = task_repo
+        self.image_id = image_id
+        self.uri = uri
+        super(_WebDownload, self).__init__(
+            name='%s-WebDownload-%s' % (task_type, task_id))
+
+        if CONF.node_staging_uri is None:
+            msg = (_("%(task_id)s of %(task_type)s not configured "
+                     "properly. Missing node_staging_uri: %(work_dir)s") %
+                   {'task_id': self.task_id,
+                    'task_type': self.task_type,
+                    'work_dir': CONF.node_staging_uri})
+            raise exception.BadTaskConfiguration(msg)
+
+        self.store = self._build_store()
+
+    def _build_store(self):
+        # NOTE(flaper87): Due to the nice glance_store api (#sarcasm), we're
+        # forced to build our own config object, register the required options
+        # (and by required I mean *ALL* of them, even the ones we don't want),
+        # and create our own store instance by calling a private function.
+        # This is certainly unfortunate but it's the best we can do until the
+        # glance_store refactor is done. A good thing is that glance_store is
+        # under our team's management and it gates on Glance so changes to
+        # this API will (should?) break task's tests.
+        conf = cfg.ConfigOpts()
+        backend.register_opts(conf)
+        conf.set_override('filesystem_store_datadir',
+                          CONF.node_staging_uri[7:],
+                          group='glance_store')
+
+        # NOTE(flaper87): Do not even try to judge me for this... :(
+        # With the glance_store refactor, this code will change, until
+        # that happens, we don't have a better option and this is the
+        # least worst one, IMHO.
+        store = backend._load_store(conf, 'file')
+
+        if store is None:
+            msg = (_("%(task_id)s of %(task_type)s not configured "
+                     "properly. Could not load the filesystem store") %
+                   {'task_id': self.task_id, 'task_type': self.task_type})
+            raise exception.BadTaskConfiguration(msg)
+
+        store.configure()
+        return store
+
+    def execute(self):
+        """Create temp file into store and return path to it
+
+        :param image_id: Glance Image ID
+        """
+        # NOTE(jokke): We've decided to use staging area for this task as
+        # a way to expect users to configure a local store for pre-import
+        # works on the image to happen.
+        #
+        # While using any path should be "technically" fine, it's not what
+        # we recommend as the best solution. For more details on this, please
+        # refer to the comment in the `_ImportToStore.execute` method.
+        data = script_utils.get_image_data_iter(self.uri)
+
+        path = self.store.add(self.image_id, data, 0)[0]
+
+        return path
+
+    def revert(self, result, **kwargs):
+        if isinstance(result, failure.Failure):
+            LOG.exception(_LE('Task: %(task_id)s failed to import image '
+                              '%(image_id)s to the filesystem.'),
+                          {'task_id': self.task_id,
+                           'image_id': self.image_id})
+
+
+def get_flow(**kwargs):
+    """Return task flow for web-download.
+
+    :param task_id: Task ID.
+    :param task_type: Type of the task.
+    :param image_repo: Image repository used.
+    :param uri: URI the image data is downloaded from.
+    """
+    task_id = kwargs.get('task_id')
+    task_type = kwargs.get('task_type')
+    image_repo = kwargs.get('image_repo')
+    image_id = kwargs.get('image_id')
+    uri = kwargs.get('import_req')['method'].get('uri')
+
+    return lf.Flow(task_type).add(
+        _WebDownload(task_id, task_type, image_repo, image_id, uri),
+    )

glance/async/flows/api_image_import.py

@@ -23,6 +23,7 @@ from taskflow.patterns import linear_flow as lf
 from taskflow import retry
 from taskflow import task
 
+import glance.async.flows._internal_plugins as internal_plugins
 import glance.async.flows.plugins as import_plugins
 from glance.common import exception
 from glance.common.scripts.image_import import main as image_import
@@ -359,16 +360,23 @@ def get_flow(**kwargs):
     task_repo = kwargs.get('task_repo')
     image_repo = kwargs.get('image_repo')
     image_id = kwargs.get('image_id')
-    uri = kwargs.get('uri')
+    import_method = kwargs.get('import_req')['method']['name']
+    uri = kwargs.get('import_req')['method'].get('uri')
 
-    if not uri:
+    if not uri and import_method == 'glance-direct':
         separator = ''
         if not CONF.node_staging_uri.endswith('/'):
             separator = '/'
         uri = separator.join((CONF.node_staging_uri, str(image_id)))
 
     flow = lf.Flow(task_type, retry=retry.AlwaysRevert())
-    flow.add(_VerifyStaging(task_id, task_type, task_repo, uri))
+
+    if uri.startswith("http") and import_method == 'web-download':
+        downloadToStaging = internal_plugins.get_import_plugin(**kwargs)
+        flow.add(downloadToStaging)
+    else:
+        file_uri = uri
+    flow.add(_VerifyStaging(task_id, task_type, task_repo, file_uri))
 
     for plugin in import_plugins.get_import_plugins(**kwargs):
         flow.add(plugin)
@@ -376,7 +384,7 @@ def get_flow(**kwargs):
     import_to_store = _ImportToStore(task_id,
                                      task_type,
                                      image_repo,
-                                     uri,
+                                     file_uri,
                                      image_id)
     flow.add(import_to_store)
 

glance/common/config.py

@@ -726,6 +726,18 @@ to Image Import Refactoring work.
 
 Related options:
     * [DEFAULT]/node_staging_uri""")),
+    cfg.ListOpt('enabled_import_methods',
+                item_type=cfg.types.String(quotes=True),
+                bounds=True,
+                default=['glance-direct', 'web-download'],
+                help=_("""
+List of enabled Image Import Methods
+
+Both 'glance-direct' and 'web-download' are enabled by default.
+
+Related options:
+    * [DEFAULT]/node_staging_uri
+    * [DEFAULT]/enable_image_import""")),
 ]
 
 CONF = cfg.CONF

glance/tests/unit/v2/test_discovery_image_import.py

@@ -37,10 +37,10 @@ class TestInfoControllers(test_utils.BaseTestCase):
     def test_get_import_info(self):
         # TODO(rosmaita): change this when import methods are
         # listed in the config file
-        import_method = 'glance-direct'
+        import_methods = ['glance-direct', 'web-download']
 
         self.config(enable_image_import=True)
         req = unit_test_utils.get_fake_request()
         output = self.controller.get_image_import(req)
         self.assertIn('import-methods', output)
-        self.assertEqual([import_method], output['import-methods']['value'])
+        self.assertEqual(import_methods, output['import-methods']['value'])

setup.cfg

@@ -76,6 +76,9 @@ glance.image_import.plugins =
     no_op = glance.async.flows.plugins.no_op:get_flow
     inject_image_metadata=glance.async.flows.plugins.inject_image_metadata:get_flow
 
+glance.image_import.internal_plugins =
+    web_download = glance.async.flows._internal_plugins.web_download:get_flow
+
 [build_sphinx]
 builder = html man
 all_files = 1