Add plug-in modules for direct downloads of glance locations

Glance can expose direct URL locations to its clients.  In current
versions of nova the only URL that can be accessed directly is file://.
This patch adds a notion of download plug-ins.  With this new download
protocol modules can be added without disruption to the rest of the
code base.  Based on the scheme of the URL returned from Glance a
plug-in will be loaded and used to download the data directly, instead
of first routing it through Glance.  If anything fails in the process
the image will be downloaded by way of Glance.

Handlers are loaded with stevedore.  To add a new module follow the example
in nova.image.downloads.file.py.  The module is required to have two
functions:

get_download_hander():
This must return a child of nova.image.download.TransferBase

get_scheme():
Return the URL scheme that this module handles (ex: 'file')

If additional configuration is needed it can be added by the specific
plug-in (as shown by file_download_module_config included with this
patch submission).  Once the module is created it must be added as an
entry point to the python installation.  For those included with nova
this can be done by adding the following it setup.cfg:

[entry_points]
nova.download.modules =
    file = nova.image.xfers.file

Additionally, as part of the multiple-locations work in Glance meta data
comes back with each location describing it.  As an example,
this is needed for direct access to file URLs.  Nova cannot assume that
every file URL is accessible on its mounted file systems, nor can it
assume that the mount points are the same.  This patch solves that
problem for direct access to files.

blueprint image-multiple-location

Change-Id: I79b863c0075cebaadce5b630f22b81d2959ddbb1

etc/nova/nova.conf.sample

@@ -2743,6 +2743,18 @@
 #sg_retry_interval=5
 
 
+[image_file_url]
+
+#
+# Options defined in nova.image.download.file
+#
+
+# A list of filesystems that will be configured in this file
+# under the sections image_file_url:<list entry name> (list
+# value)
+#filesystems=
+
+
 [baremetal]
 
 #

nova/exception.py

@@ -1331,3 +1331,30 @@ class InstanceGroupPolicyNotFound(NotFound):
 
 class PluginRetriesExceeded(NovaException):
     msg_fmt = _("Number of retries to plugin (%(num_retries)d) exceeded.")
+
+
+class ImageDownloadModuleError(NovaException):
+    msg_fmt = _("There was an error with the download module %(module)s. "
+                "%(reason)s")
+
+
+class ImageDownloadModuleLoadError(ImageDownloadModuleError):
+    msg_fmt = _("Could not load the module %(module)s")
+
+
+class ImageDownloadModuleMetaDataError(ImageDownloadModuleError):
+    msg_fmt = _("The metadata for this location will not work with this "
+                "module %(module)s.  %(reason)s.")
+
+
+class ImageDownloadModuleNotImplementedError(ImageDownloadModuleError):
+    msg_fmt = _("The method %(method_name)s is not implemented.")
+
+
+class ImageDownloadModuleMetaDataError(ImageDownloadModuleError):
+    msg_fmt = _("The metadata for this location will not work with this "
+                "module %(module)s.  %(reason)s.")
+
+
+class ImageDownloadModuleConfigurationError(ImageDownloadModuleError):
+    msg_fmt = _("The module %(module)s is misconfigured: %(reason)s.")

nova/image/download/__init__.py

@@ -0,0 +1,50 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+
+from oslo.config import cfg
+import stevedore.driver
+import stevedore.extension
+
+from nova.openstack.common.gettextutils import _
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+
+def load_transfer_modules():
+
+    module_dictionary = {}
+
+    ex = stevedore.extension.ExtensionManager('nova.image.download.modules')
+    for module_name in ex.names():
+        mgr = stevedore.driver.DriverManager(
+            namespace='nova.image.download.modules',
+            name=module_name,
+            invoke_on_load=False)
+
+        schemes_list = mgr.driver.get_schemes()
+        for scheme in schemes_list:
+            if scheme in module_dictionary:
+                msg = _('%(scheme)s is registered as a module twice. '
+                        '%(module_name)s is not being used.')
+                LOG.error(msg)
+            else:
+                module_dictionary[scheme] = mgr.driver
+
+    return module_dictionary

nova/image/download/base.py

@@ -0,0 +1,25 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nova import exception
+
+
+class TransferBase(object):
+
+    def download(self, url_parts, destination, metadata, **kwargs):
+        raise exception.ImageDownloadModuleNotImplementedError(
+            method_name='download')

nova/image/download/file.py

@@ -0,0 +1,169 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+import shutil
+
+from oslo.config import cfg
+
+from nova import exception
+import nova.image.download.base as xfer_base
+from nova.openstack.common.gettextutils import _
+
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+opt_group = cfg.ListOpt(name='filesystems', default=[],
+                        help=_('A list of filesystems that will be configured '
+                               'in this file under the sections '
+                               'image_file_url:<list entry name>'))
+CONF.register_opt(opt_group, group="image_file_url")
+
+
+#  This module extends the configuration options for nova.conf.  If the user
+#  wishes to use the specific configuration settings the following needs to
+#  be added to nova.conf:
+#  [image_file_url]
+#  filesystem = <a list of strings referencing a config section>
+#
+#  For each entry in the filesystem list a new configuration section must be
+#  added with the following format:
+#  [image_file_url:<list entry>]
+#  id = <string>
+#  mountpoint = <string>
+#
+#    id:
+#        An opaque string.  In order for this module to know that the remote
+#        FS is the same one that is mounted locally it must share information
+#        with the glance deployment.  Both glance and nova-compute must be
+#        configured with a unique matching string.  This ensures that the
+#        file:// advertised URL is describing a file system that is known
+#        to nova-compute
+#    mountpoint:
+#        The location at which the file system is locally mounted.  Glance
+#        may mount a shared file system on a different path than nova-compute.
+#        This value will be compared against the metadata advertised with
+#        glance and paths will be adjusted to ensure that the correct file
+#        file is copied.
+#
+#  If these values are not added to nova.conf and the file module is in the
+#  allowed_direct_url_schemes list, then the legacy behavior will occur such
+#  that a copy will be attempted assuming that the glance and nova file systems
+#  are the same.
+
+
+class FileTransfer(xfer_base.TransferBase):
+
+    desc_required_keys = ['id', 'mountpoint']
+
+    #NOTE(jbresnah) because the group under which these options are added is
+    # dyncamically determined these options need to stay out of global space
+    # or they will confuse generate_sample.sh
+    filesystem_opts = [
+         cfg.StrOpt('id',
+                    help=_('A unique ID given to each file system.  This is '
+                           'value is set in Glance and agreed upon here so '
+                           'that the operator knowns they are dealing with '
+                           'the same file system.')),
+         cfg.StrOpt('mountpoint',
+                    help=_('The path at which the file system is mounted.')),
+    ]
+
+    def _get_options(self):
+        fs_dict = {}
+        for fs in CONF.image_file_url.filesystems:
+            group_name = 'image_file_url:' + fs
+            conf_group = CONF[group_name]
+            if conf_group.id is None:
+                msg = _('The group %s(group_name) must be configured with '
+                        'an id.')
+                raise exception.ImageDownloadModuleConfigurationError(
+                    module=str(self), reason=msg)
+            fs_dict[CONF[group_name].id] = CONF[group_name]
+        return fs_dict
+
+    def __init__(self):
+        # create the needed options
+        for fs in CONF.image_file_url.filesystems:
+            group_name = 'image_file_url:' + fs
+            CONF.register_opts(self.filesystem_opts, group=group_name)
+
+    def _verify_config(self):
+        for fs_key in self.filesystems:
+            for r in self.desc_required_keys:
+                fs_ent = self.filesystems[fs_key]
+                if fs_ent[r] is None:
+                    msg = _('The key %s is required in all file system '
+                            'descriptions.')
+                    LOG.error(msg)
+                    raise exception.ImageDownloadModuleConfigurationError(
+                        module=str(self), reason=msg)
+
+    def _file_system_lookup(self, metadata, url_parts):
+        for r in self.desc_required_keys:
+            if r not in metadata:
+                url = url_parts.geturl()
+                msg = _('The key %(r)s is required in the location metadata '
+                        'to access the url %(url)s.') % locals()
+                LOG.info(msg)
+                raise exception.ImageDownloadModuleMetaDataError(
+                    module=str(self), reason=msg)
+        fs_descriptor = self.filesystems[metadata['id']]
+        return fs_descriptor
+
+    def _normalize_destination(self, nova_mount, glance_mount, path):
+        if not path.startswith(glance_mount):
+            msg = _('The mount point advertised by glance: %(glance_mount)s, '
+                    'does not match the URL path: %(path)s') % locals()
+            raise exception.ImageDownloadModuleMetaDataError(
+                module=str(self), reason=msg)
+        new_path = path.replace(glance_mount, nova_mount, 1)
+        return new_path
+
+    def download(self, url_parts, destination, metadata, **kwargs):
+        self.filesystems = self._get_options()
+        if not self.filesystems:
+            #NOTE(jbresnah) when nothing is configured assume legacy behavior
+            nova_mountpoint = '/'
+            glance_mountpoint = '/'
+        else:
+            self._verify_config()
+            fs_descriptor = self._file_system_lookup(metadata, url_parts)
+            if fs_descriptor is None:
+                msg = (_('No matching ID for the URL %s was found.') %
+                        url_parts.geturl())
+                raise exception.ImageDownloadModuleError(reason=msg,
+                                                    module=str(self))
+            nova_mountpoint = fs_descriptor['mountpoint']
+            glance_mountpoint = metadata['mountpoint']
+
+        source_file = self._normalize_destination(nova_mountpoint,
+                                                  glance_mountpoint,
+                                                  url_parts.path)
+        with open(source_file, "r") as f:
+            shutil.copyfileobj(f, destination)
+            LOG.info(_('Copied %(source_file)s using %(module_str)s') %
+                     {'source_file': source_file, 'module_str': str(self)})
+
+
+def get_download_hander(**kwargs):
+    return FileTransfer()
+
+
+def get_schemes():
+    return ['file', 'filesystem']

nova/image/glance.py

@@ -23,7 +23,6 @@ import copy
 import itertools
 import json
 import random
-import shutil
 import sys
 import time
 import urlparse
@@ -33,6 +32,7 @@ import glanceclient.exc
 from oslo.config import cfg
 
 from nova import exception
+import nova.image.download as image_xfers
 from nova.openstack.common.gettextutils import _
 from nova.openstack.common import jsonutils
 from nova.openstack.common import log as logging
@@ -74,6 +74,8 @@ CONF.register_opts(glance_opts)
 CONF.import_opt('auth_strategy', 'nova.api.auth')
 CONF.import_opt('my_ip', 'nova.netconf')
 
+_DOWNLOAD_MODULES = image_xfers.load_transfer_modules()
+
 
 def generate_glance_url():
     """Generate the URL to glance."""
@@ -220,6 +222,22 @@ class GlanceImageService(object):
 
     def __init__(self, client=None):
         self._client = client or GlanceClientWrapper()
+        #NOTE(jbresnah) build the table of download handlers at the beginning
+        # so that operators can catch errors at load time rather than whenever
+        # a user attempts to use a module.  Note this cannot be done in glance
+        # space when this python module is loaded because the download module
+        # may require configuration options to be parsed.
+        self._download_handlers = {}
+        for scheme in _DOWNLOAD_MODULES:
+            if scheme in CONF.allowed_direct_url_schemes:
+                mod = _DOWNLOAD_MODULES[scheme]
+                try:
+                    self._download_handlers[scheme] = mod.get_download_hander()
+                except Exception as ex:
+                    msg = _('When loading the module %(module_str)s the '
+                            'following error occurred: %(ex)s')\
+                            % {'module_str': str(mod), 'ex': ex}
+                    LOG.error(msg)
 
     def detail(self, context, **kwargs):
         """Calls out to Glance for a list of detailed image information."""
@@ -264,7 +282,7 @@ class GlanceImageService(object):
         base_image_meta = self._translate_from_glance(image)
         return base_image_meta
 
-    def get_location(self, context, image_id):
+    def _get_locations(self, context, image_id):
         """Returns the direct url representing the backend storage location,
         or None if this attribute is not shown by Glance.
         """
@@ -277,21 +295,37 @@ class GlanceImageService(object):
         if not self._is_image_available(context, image_meta):
             raise exception.ImageNotFound(image_id=image_id)
 
-        return getattr(image_meta, 'direct_url', None)
+        locations = getattr(image_meta, 'locations', [])
+        du = getattr(image_meta, 'direct_url', None)
+        if du:
+            locations.append({'url': du, 'metadata': {}})
+        return locations
+
+    def _get_transfer_module(self, scheme):
+        try:
+            return self._download_handlers[scheme]
+        except KeyError:
+            return None
+        except Exception as ex:
+            LOG.error(_("Failed to instantiate the download handler "
+                        "for %(scheme)s") % locals())
+        return
 
     def download(self, context, image_id, data=None):
         """Calls out to Glance for data and writes data."""
-        if 'file' in CONF.allowed_direct_url_schemes:
+        locations = self._get_locations(context, image_id)
-            location = self.get_location(context, image_id)
+        for entry in locations:
-            o = urlparse.urlparse(location)
+            loc_url = entry['url']
-            if o.scheme == "file":
+            loc_meta = entry['metadata']
-                with open(o.path, "r") as f:
+            o = urlparse.urlparse(loc_url)
-                    # FIXME(jbresnah) a system call to cp could have
+            xfer_mod = self._get_transfer_module(o.scheme)
-                    # significant performance advantages, however we
+            if xfer_mod:
-                    # do not have the path to files at this point in
+                try:
-                    # the abstraction.
+                    xfer_mod.download(o, data, loc_meta)
-                    shutil.copyfileobj(f, data)
+                    LOG.info("Successfully transferred using %s" % o.scheme)
                     return
+                except Exception as ex:
+                    LOG.exception(ex)
 
         try:
             image_chunks = self._client.call(context, 1, 'data', image_id)

nova/tests/image/test_glance.py

@@ -20,9 +20,12 @@ import datetime
 import filecmp
 import os
 import random
+import shutil
 import tempfile
 import time
 
+import mox
+
 import glanceclient.exc
 from oslo.config import cfg
 
@@ -110,6 +113,22 @@ class TestGlanceImageService(test.TestCase):
         client = glance_stubs.StubGlanceClient()
         self.service = self._create_image_service(client)
         self.context = context.RequestContext('fake', 'fake', auth_token=True)
+        self.mox = mox.Mox()
+        self.files_to_clean = []
+
+    def tearDown(self):
+        super(TestGlanceImageService, self).tearDown()
+        self.mox.UnsetStubs()
+        for f in self.files_to_clean:
+            try:
+                os.unlink(f)
+            except Exception:
+                pass
+
+    def _get_tempfile(self):
+        (outfd, config_filename) = tempfile.mkstemp(prefix='nova_glance_tests')
+        self.files_to_clean.append(config_filename)
+        return (outfd, config_filename)
 
     def _create_image_service(self, client):
         def _fake_create_glance_client(context, host, port, use_ssl, version):
@@ -479,6 +498,8 @@ class TestGlanceImageService(test.TestCase):
         service.download(self.context, image_id, writer)
 
     def test_download_file_url(self):
+        self.flags(allowed_direct_url_schemes=['file'])
+
         class MyGlanceStubClient(glance_stubs.StubGlanceClient):
             """A client that returns a file url."""
 
@@ -501,7 +522,6 @@ class TestGlanceImageService(test.TestCase):
         service = self._create_image_service(client)
         image_id = 1  # doesn't matter
 
-        self.flags(allowed_direct_url_schemes=['file'])
         service.download(self.context, image_id, writer)
         writer.close()
 
@@ -512,6 +532,149 @@ class TestGlanceImageService(test.TestCase):
         os.remove(client.s_tmpfname)
         os.remove(tmpfname)
 
+    def test_download_module_filesystem_match(self):
+
+        mountpoint = '/'
+        fs_id = 'someid'
+        desc = {'id': fs_id, 'mountpoint': mountpoint}
+
+        class MyGlanceStubClient(glance_stubs.StubGlanceClient):
+            outer_test = self
+
+            def get(self, image_id):
+                return type('GlanceLocations', (object,),
+                            {'locations': [
+                                {'url': 'file:///' + os.devnull,
+                                 'metadata': desc}]})
+
+            def data(self, image_id):
+                self.outer_test.fail('This should not be called because the '
+                                     'transfer module should have intercepted '
+                                     'it.')
+
+        self.mox.StubOutWithMock(shutil, 'copyfileobj')
+
+        image_id = 1  # doesn't matter
+        client = MyGlanceStubClient()
+        self.flags(allowed_direct_url_schemes=['file'])
+        self.flags(group='image_file_url', filesystems=['gluster'])
+        service = self._create_image_service(client)
+        #NOTE(Jbresnah) The following options must be added after the module
+        # has added the specific groups.
+        self.flags(group='image_file_url:gluster', id=fs_id)
+        self.flags(group='image_file_url:gluster', mountpoint=mountpoint)
+
+        shutil.copyfileobj(mox.IgnoreArg(), mox.IgnoreArg())
+
+        self.mox.ReplayAll()
+        service.download(self.context, image_id)
+        self.mox.VerifyAll()
+
+    def test_download_module_no_filesystem_match(self):
+        mountpoint = '/'
+        fs_id = 'someid'
+        desc = {'id': fs_id, 'mountpoint': mountpoint}
+        some_data = "sfxvdwjer"
+
+        class MyGlanceStubClient(glance_stubs.StubGlanceClient):
+            outer_test = self
+
+            def get(self, image_id):
+                return type('GlanceLocations', (object,),
+                            {'locations': [
+                                {'url': 'file:///' + os.devnull,
+                                 'metadata': desc}]})
+
+            def data(self, image_id):
+                return some_data
+
+        def _fake_copyfileobj(source, dest):
+            self.fail('This should not be called because a match should not '
+                      'have been found.')
+        self.stubs.Set(shutil, 'copyfileobj', _fake_copyfileobj)
+
+        image_id = 1  # doesn't matter
+        client = MyGlanceStubClient()
+        self.flags(allowed_direct_url_schemes=['file'])
+        self.flags(group='image_file_url', filesystems=['gluster'])
+        service = self._create_image_service(client)
+        #NOTE(Jbresnah) The following options must be added after the module
+        # has added the specific groups.
+        self.flags(group='image_file_url:gluster', id='someotherid')
+        self.flags(group='image_file_url:gluster', mountpoint=mountpoint)
+
+        rc = service.download(self.context, image_id)
+        self.assertEqual(some_data, rc)
+
+    def test_download_module_mountpoints(self):
+        glance_mount = '/glance/mount/point'
+        _, data_filename = self._get_tempfile()
+        nova_mount = os.path.dirname(data_filename)
+        source_path = os.path.basename(data_filename)
+        file_url = 'file://%s' % os.path.join(glance_mount, source_path)
+        file_system_id = 'test_FS_ID'
+        file_system_desc = {'id': file_system_id, 'mountpoint': glance_mount}
+
+        class MyGlanceStubClient(glance_stubs.StubGlanceClient):
+            outer_test = self
+
+            def get(self, image_id):
+                return type('GlanceLocations', (object,),
+                            {'locations': [{'url': file_url,
+                                            'metadata': file_system_desc}]})
+
+            def data(self, image_id):
+                self.outer_test.fail('This should not be called because the '
+                                     'transfer module should have intercepted '
+                                     'it.')
+
+        self.copy_called = False
+
+        def _fake_copyfileobj(source, dest):
+            self.assertEqual(source.name, data_filename)
+            self.copy_called = True
+        self.stubs.Set(shutil, 'copyfileobj', _fake_copyfileobj)
+
+        self.flags(allowed_direct_url_schemes=['file'])
+        self.flags(group='image_file_url', filesystems=['gluster'])
+        image_id = 1  # doesn't matter
+        client = MyGlanceStubClient()
+        service = self._create_image_service(client)
+        self.flags(group='image_file_url:gluster', id=file_system_id)
+        self.flags(group='image_file_url:gluster', mountpoint=nova_mount)
+
+        service.download(self.context, image_id)
+        self.assertTrue(self.copy_called)
+
+    def test_download_module_file_bad_module(self):
+        _, data_filename = self._get_tempfile()
+        file_url = 'applesauce://%s' % data_filename
+        data_from_client = "someData"
+
+        class MyGlanceStubClient(glance_stubs.StubGlanceClient):
+            def get(self, image_id):
+                return type('GlanceLocations', (object,),
+                            {'locations': [{'url': file_url,
+                                            'metadata': {}}]})
+
+            def data(self, image_id):
+                return data_from_client
+
+        self.flags(allowed_direct_url_schemes=['applesauce'])
+
+        self.mox.StubOutWithMock(shutil, 'copyfileobj')
+        self.flags(allowed_direct_url_schemes=['file'])
+        image_id = 1  # doesn't matter
+        client = MyGlanceStubClient()
+        service = self._create_image_service(client)
+
+        # by not calling copyfileobj in the file download module we verify
+        # that the requirements were not met for its use
+        self.mox.ReplayAll()
+        new_data = service.download(self.context, image_id)
+        self.mox.VerifyAll()
+        self.assertEqual(data_from_client, new_data)
+
     def test_client_forbidden_converts_to_imagenotauthed(self):
         class MyGlanceStubClient(glance_stubs.StubGlanceClient):
             """A client that raises a Forbidden exception."""

setup.cfg

@@ -27,6 +27,8 @@ packages =
     nova
 
 [entry_points]
+nova.image.download.modules =
+    file = nova.image.download.file
 console_scripts =
     nova-all = nova.cmd.all:main
     nova-api = nova.cmd.api:main