From 6f9ed562d5f6df1dc16029dcaffd4d2e6601099d Mon Sep 17 00:00:00 2001 From: John Bresnahan Date: Thu, 18 Jul 2013 10:39:33 -1000 Subject: [PATCH] Add plug-in modules for direct downloads of glance locations Glance can expose direct URL locations to its clients. In current versions of nova the only URL that can be accessed directly is file://. This patch adds a notion of download plug-ins. With this new download protocol modules can be added without disruption to the rest of the code base. Based on the scheme of the URL returned from Glance a plug-in will be loaded and used to download the data directly, instead of first routing it through Glance. If anything fails in the process the image will be downloaded by way of Glance. Handlers are loaded with stevedore. To add a new module follow the example in nova.image.downloads.file.py. The module is required to have two functions: get_download_hander(): This must return a child of nova.image.download.TransferBase get_scheme(): Return the URL scheme that this module handles (ex: 'file') If additional configuration is needed it can be added by the specific plug-in (as shown by file_download_module_config included with this patch submission). Once the module is created it must be added as an entry point to the python installation. For those included with nova this can be done by adding the following it setup.cfg: [entry_points] nova.download.modules = file = nova.image.xfers.file Additionally, as part of the multiple-locations work in Glance meta data comes back with each location describing it. As an example, this is needed for direct access to file URLs. Nova cannot assume that every file URL is accessible on its mounted file systems, nor can it assume that the mount points are the same. This patch solves that problem for direct access to files. blueprint image-multiple-location Change-Id: I79b863c0075cebaadce5b630f22b81d2959ddbb1 --- etc/nova/nova.conf.sample | 12 +++ nova/exception.py | 27 +++++ nova/image/download/__init__.py | 50 ++++++++++ nova/image/download/base.py | 25 +++++ nova/image/download/file.py | 169 ++++++++++++++++++++++++++++++++ nova/image/glance.py | 62 +++++++++--- nova/tests/image/test_glance.py | 165 ++++++++++++++++++++++++++++++- setup.cfg | 2 + 8 files changed, 497 insertions(+), 15 deletions(-) create mode 100644 nova/image/download/__init__.py create mode 100644 nova/image/download/base.py create mode 100644 nova/image/download/file.py diff --git a/etc/nova/nova.conf.sample b/etc/nova/nova.conf.sample index 362b564450cc..93ae85e24c31 100644 --- a/etc/nova/nova.conf.sample +++ b/etc/nova/nova.conf.sample @@ -2743,6 +2743,18 @@ #sg_retry_interval=5 +[image_file_url] + +# +# Options defined in nova.image.download.file +# + +# A list of filesystems that will be configured in this file +# under the sections image_file_url: (list +# value) +#filesystems= + + [baremetal] # diff --git a/nova/exception.py b/nova/exception.py index 49dd52b22585..3c60295147c9 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -1331,3 +1331,30 @@ class InstanceGroupPolicyNotFound(NotFound): class PluginRetriesExceeded(NovaException): msg_fmt = _("Number of retries to plugin (%(num_retries)d) exceeded.") + + +class ImageDownloadModuleError(NovaException): + msg_fmt = _("There was an error with the download module %(module)s. " + "%(reason)s") + + +class ImageDownloadModuleLoadError(ImageDownloadModuleError): + msg_fmt = _("Could not load the module %(module)s") + + +class ImageDownloadModuleMetaDataError(ImageDownloadModuleError): + msg_fmt = _("The metadata for this location will not work with this " + "module %(module)s. %(reason)s.") + + +class ImageDownloadModuleNotImplementedError(ImageDownloadModuleError): + msg_fmt = _("The method %(method_name)s is not implemented.") + + +class ImageDownloadModuleMetaDataError(ImageDownloadModuleError): + msg_fmt = _("The metadata for this location will not work with this " + "module %(module)s. %(reason)s.") + + +class ImageDownloadModuleConfigurationError(ImageDownloadModuleError): + msg_fmt = _("The module %(module)s is misconfigured: %(reason)s.") diff --git a/nova/image/download/__init__.py b/nova/image/download/__init__.py new file mode 100644 index 000000000000..c05688387660 --- /dev/null +++ b/nova/image/download/__init__.py @@ -0,0 +1,50 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from oslo.config import cfg +import stevedore.driver +import stevedore.extension + +from nova.openstack.common.gettextutils import _ + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +def load_transfer_modules(): + + module_dictionary = {} + + ex = stevedore.extension.ExtensionManager('nova.image.download.modules') + for module_name in ex.names(): + mgr = stevedore.driver.DriverManager( + namespace='nova.image.download.modules', + name=module_name, + invoke_on_load=False) + + schemes_list = mgr.driver.get_schemes() + for scheme in schemes_list: + if scheme in module_dictionary: + msg = _('%(scheme)s is registered as a module twice. ' + '%(module_name)s is not being used.') + LOG.error(msg) + else: + module_dictionary[scheme] = mgr.driver + + return module_dictionary diff --git a/nova/image/download/base.py b/nova/image/download/base.py new file mode 100644 index 000000000000..2e125eeb5d6d --- /dev/null +++ b/nova/image/download/base.py @@ -0,0 +1,25 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import exception + + +class TransferBase(object): + + def download(self, url_parts, destination, metadata, **kwargs): + raise exception.ImageDownloadModuleNotImplementedError( + method_name='download') diff --git a/nova/image/download/file.py b/nova/image/download/file.py new file mode 100644 index 000000000000..f3ac7da85cfb --- /dev/null +++ b/nova/image/download/file.py @@ -0,0 +1,169 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import shutil + +from oslo.config import cfg + +from nova import exception +import nova.image.download.base as xfer_base +from nova.openstack.common.gettextutils import _ + + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + +opt_group = cfg.ListOpt(name='filesystems', default=[], + help=_('A list of filesystems that will be configured ' + 'in this file under the sections ' + 'image_file_url:')) +CONF.register_opt(opt_group, group="image_file_url") + + +# This module extends the configuration options for nova.conf. If the user +# wishes to use the specific configuration settings the following needs to +# be added to nova.conf: +# [image_file_url] +# filesystem = +# +# For each entry in the filesystem list a new configuration section must be +# added with the following format: +# [image_file_url:] +# id = +# mountpoint = +# +# id: +# An opaque string. In order for this module to know that the remote +# FS is the same one that is mounted locally it must share information +# with the glance deployment. Both glance and nova-compute must be +# configured with a unique matching string. This ensures that the +# file:// advertised URL is describing a file system that is known +# to nova-compute +# mountpoint: +# The location at which the file system is locally mounted. Glance +# may mount a shared file system on a different path than nova-compute. +# This value will be compared against the metadata advertised with +# glance and paths will be adjusted to ensure that the correct file +# file is copied. +# +# If these values are not added to nova.conf and the file module is in the +# allowed_direct_url_schemes list, then the legacy behavior will occur such +# that a copy will be attempted assuming that the glance and nova file systems +# are the same. + + +class FileTransfer(xfer_base.TransferBase): + + desc_required_keys = ['id', 'mountpoint'] + + #NOTE(jbresnah) because the group under which these options are added is + # dyncamically determined these options need to stay out of global space + # or they will confuse generate_sample.sh + filesystem_opts = [ + cfg.StrOpt('id', + help=_('A unique ID given to each file system. This is ' + 'value is set in Glance and agreed upon here so ' + 'that the operator knowns they are dealing with ' + 'the same file system.')), + cfg.StrOpt('mountpoint', + help=_('The path at which the file system is mounted.')), + ] + + def _get_options(self): + fs_dict = {} + for fs in CONF.image_file_url.filesystems: + group_name = 'image_file_url:' + fs + conf_group = CONF[group_name] + if conf_group.id is None: + msg = _('The group %s(group_name) must be configured with ' + 'an id.') + raise exception.ImageDownloadModuleConfigurationError( + module=str(self), reason=msg) + fs_dict[CONF[group_name].id] = CONF[group_name] + return fs_dict + + def __init__(self): + # create the needed options + for fs in CONF.image_file_url.filesystems: + group_name = 'image_file_url:' + fs + CONF.register_opts(self.filesystem_opts, group=group_name) + + def _verify_config(self): + for fs_key in self.filesystems: + for r in self.desc_required_keys: + fs_ent = self.filesystems[fs_key] + if fs_ent[r] is None: + msg = _('The key %s is required in all file system ' + 'descriptions.') + LOG.error(msg) + raise exception.ImageDownloadModuleConfigurationError( + module=str(self), reason=msg) + + def _file_system_lookup(self, metadata, url_parts): + for r in self.desc_required_keys: + if r not in metadata: + url = url_parts.geturl() + msg = _('The key %(r)s is required in the location metadata ' + 'to access the url %(url)s.') % locals() + LOG.info(msg) + raise exception.ImageDownloadModuleMetaDataError( + module=str(self), reason=msg) + fs_descriptor = self.filesystems[metadata['id']] + return fs_descriptor + + def _normalize_destination(self, nova_mount, glance_mount, path): + if not path.startswith(glance_mount): + msg = _('The mount point advertised by glance: %(glance_mount)s, ' + 'does not match the URL path: %(path)s') % locals() + raise exception.ImageDownloadModuleMetaDataError( + module=str(self), reason=msg) + new_path = path.replace(glance_mount, nova_mount, 1) + return new_path + + def download(self, url_parts, destination, metadata, **kwargs): + self.filesystems = self._get_options() + if not self.filesystems: + #NOTE(jbresnah) when nothing is configured assume legacy behavior + nova_mountpoint = '/' + glance_mountpoint = '/' + else: + self._verify_config() + fs_descriptor = self._file_system_lookup(metadata, url_parts) + if fs_descriptor is None: + msg = (_('No matching ID for the URL %s was found.') % + url_parts.geturl()) + raise exception.ImageDownloadModuleError(reason=msg, + module=str(self)) + nova_mountpoint = fs_descriptor['mountpoint'] + glance_mountpoint = metadata['mountpoint'] + + source_file = self._normalize_destination(nova_mountpoint, + glance_mountpoint, + url_parts.path) + with open(source_file, "r") as f: + shutil.copyfileobj(f, destination) + LOG.info(_('Copied %(source_file)s using %(module_str)s') % + {'source_file': source_file, 'module_str': str(self)}) + + +def get_download_hander(**kwargs): + return FileTransfer() + + +def get_schemes(): + return ['file', 'filesystem'] diff --git a/nova/image/glance.py b/nova/image/glance.py index f017b863b45f..9a691bf53d1d 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -23,7 +23,6 @@ import copy import itertools import json import random -import shutil import sys import time import urlparse @@ -33,6 +32,7 @@ import glanceclient.exc from oslo.config import cfg from nova import exception +import nova.image.download as image_xfers from nova.openstack.common.gettextutils import _ from nova.openstack.common import jsonutils from nova.openstack.common import log as logging @@ -74,6 +74,8 @@ CONF.register_opts(glance_opts) CONF.import_opt('auth_strategy', 'nova.api.auth') CONF.import_opt('my_ip', 'nova.netconf') +_DOWNLOAD_MODULES = image_xfers.load_transfer_modules() + def generate_glance_url(): """Generate the URL to glance.""" @@ -220,6 +222,22 @@ class GlanceImageService(object): def __init__(self, client=None): self._client = client or GlanceClientWrapper() + #NOTE(jbresnah) build the table of download handlers at the beginning + # so that operators can catch errors at load time rather than whenever + # a user attempts to use a module. Note this cannot be done in glance + # space when this python module is loaded because the download module + # may require configuration options to be parsed. + self._download_handlers = {} + for scheme in _DOWNLOAD_MODULES: + if scheme in CONF.allowed_direct_url_schemes: + mod = _DOWNLOAD_MODULES[scheme] + try: + self._download_handlers[scheme] = mod.get_download_hander() + except Exception as ex: + msg = _('When loading the module %(module_str)s the ' + 'following error occurred: %(ex)s')\ + % {'module_str': str(mod), 'ex': ex} + LOG.error(msg) def detail(self, context, **kwargs): """Calls out to Glance for a list of detailed image information.""" @@ -264,7 +282,7 @@ class GlanceImageService(object): base_image_meta = self._translate_from_glance(image) return base_image_meta - def get_location(self, context, image_id): + def _get_locations(self, context, image_id): """Returns the direct url representing the backend storage location, or None if this attribute is not shown by Glance. """ @@ -277,21 +295,37 @@ class GlanceImageService(object): if not self._is_image_available(context, image_meta): raise exception.ImageNotFound(image_id=image_id) - return getattr(image_meta, 'direct_url', None) + locations = getattr(image_meta, 'locations', []) + du = getattr(image_meta, 'direct_url', None) + if du: + locations.append({'url': du, 'metadata': {}}) + return locations + + def _get_transfer_module(self, scheme): + try: + return self._download_handlers[scheme] + except KeyError: + return None + except Exception as ex: + LOG.error(_("Failed to instantiate the download handler " + "for %(scheme)s") % locals()) + return def download(self, context, image_id, data=None): """Calls out to Glance for data and writes data.""" - if 'file' in CONF.allowed_direct_url_schemes: - location = self.get_location(context, image_id) - o = urlparse.urlparse(location) - if o.scheme == "file": - with open(o.path, "r") as f: - # FIXME(jbresnah) a system call to cp could have - # significant performance advantages, however we - # do not have the path to files at this point in - # the abstraction. - shutil.copyfileobj(f, data) - return + locations = self._get_locations(context, image_id) + for entry in locations: + loc_url = entry['url'] + loc_meta = entry['metadata'] + o = urlparse.urlparse(loc_url) + xfer_mod = self._get_transfer_module(o.scheme) + if xfer_mod: + try: + xfer_mod.download(o, data, loc_meta) + LOG.info("Successfully transferred using %s" % o.scheme) + return + except Exception as ex: + LOG.exception(ex) try: image_chunks = self._client.call(context, 1, 'data', image_id) diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py index b1a039461c9c..7cc924e99510 100644 --- a/nova/tests/image/test_glance.py +++ b/nova/tests/image/test_glance.py @@ -20,9 +20,12 @@ import datetime import filecmp import os import random +import shutil import tempfile import time +import mox + import glanceclient.exc from oslo.config import cfg @@ -110,6 +113,22 @@ class TestGlanceImageService(test.TestCase): client = glance_stubs.StubGlanceClient() self.service = self._create_image_service(client) self.context = context.RequestContext('fake', 'fake', auth_token=True) + self.mox = mox.Mox() + self.files_to_clean = [] + + def tearDown(self): + super(TestGlanceImageService, self).tearDown() + self.mox.UnsetStubs() + for f in self.files_to_clean: + try: + os.unlink(f) + except Exception: + pass + + def _get_tempfile(self): + (outfd, config_filename) = tempfile.mkstemp(prefix='nova_glance_tests') + self.files_to_clean.append(config_filename) + return (outfd, config_filename) def _create_image_service(self, client): def _fake_create_glance_client(context, host, port, use_ssl, version): @@ -479,6 +498,8 @@ class TestGlanceImageService(test.TestCase): service.download(self.context, image_id, writer) def test_download_file_url(self): + self.flags(allowed_direct_url_schemes=['file']) + class MyGlanceStubClient(glance_stubs.StubGlanceClient): """A client that returns a file url.""" @@ -501,7 +522,6 @@ class TestGlanceImageService(test.TestCase): service = self._create_image_service(client) image_id = 1 # doesn't matter - self.flags(allowed_direct_url_schemes=['file']) service.download(self.context, image_id, writer) writer.close() @@ -512,6 +532,149 @@ class TestGlanceImageService(test.TestCase): os.remove(client.s_tmpfname) os.remove(tmpfname) + def test_download_module_filesystem_match(self): + + mountpoint = '/' + fs_id = 'someid' + desc = {'id': fs_id, 'mountpoint': mountpoint} + + class MyGlanceStubClient(glance_stubs.StubGlanceClient): + outer_test = self + + def get(self, image_id): + return type('GlanceLocations', (object,), + {'locations': [ + {'url': 'file:///' + os.devnull, + 'metadata': desc}]}) + + def data(self, image_id): + self.outer_test.fail('This should not be called because the ' + 'transfer module should have intercepted ' + 'it.') + + self.mox.StubOutWithMock(shutil, 'copyfileobj') + + image_id = 1 # doesn't matter + client = MyGlanceStubClient() + self.flags(allowed_direct_url_schemes=['file']) + self.flags(group='image_file_url', filesystems=['gluster']) + service = self._create_image_service(client) + #NOTE(Jbresnah) The following options must be added after the module + # has added the specific groups. + self.flags(group='image_file_url:gluster', id=fs_id) + self.flags(group='image_file_url:gluster', mountpoint=mountpoint) + + shutil.copyfileobj(mox.IgnoreArg(), mox.IgnoreArg()) + + self.mox.ReplayAll() + service.download(self.context, image_id) + self.mox.VerifyAll() + + def test_download_module_no_filesystem_match(self): + mountpoint = '/' + fs_id = 'someid' + desc = {'id': fs_id, 'mountpoint': mountpoint} + some_data = "sfxvdwjer" + + class MyGlanceStubClient(glance_stubs.StubGlanceClient): + outer_test = self + + def get(self, image_id): + return type('GlanceLocations', (object,), + {'locations': [ + {'url': 'file:///' + os.devnull, + 'metadata': desc}]}) + + def data(self, image_id): + return some_data + + def _fake_copyfileobj(source, dest): + self.fail('This should not be called because a match should not ' + 'have been found.') + self.stubs.Set(shutil, 'copyfileobj', _fake_copyfileobj) + + image_id = 1 # doesn't matter + client = MyGlanceStubClient() + self.flags(allowed_direct_url_schemes=['file']) + self.flags(group='image_file_url', filesystems=['gluster']) + service = self._create_image_service(client) + #NOTE(Jbresnah) The following options must be added after the module + # has added the specific groups. + self.flags(group='image_file_url:gluster', id='someotherid') + self.flags(group='image_file_url:gluster', mountpoint=mountpoint) + + rc = service.download(self.context, image_id) + self.assertEqual(some_data, rc) + + def test_download_module_mountpoints(self): + glance_mount = '/glance/mount/point' + _, data_filename = self._get_tempfile() + nova_mount = os.path.dirname(data_filename) + source_path = os.path.basename(data_filename) + file_url = 'file://%s' % os.path.join(glance_mount, source_path) + file_system_id = 'test_FS_ID' + file_system_desc = {'id': file_system_id, 'mountpoint': glance_mount} + + class MyGlanceStubClient(glance_stubs.StubGlanceClient): + outer_test = self + + def get(self, image_id): + return type('GlanceLocations', (object,), + {'locations': [{'url': file_url, + 'metadata': file_system_desc}]}) + + def data(self, image_id): + self.outer_test.fail('This should not be called because the ' + 'transfer module should have intercepted ' + 'it.') + + self.copy_called = False + + def _fake_copyfileobj(source, dest): + self.assertEqual(source.name, data_filename) + self.copy_called = True + self.stubs.Set(shutil, 'copyfileobj', _fake_copyfileobj) + + self.flags(allowed_direct_url_schemes=['file']) + self.flags(group='image_file_url', filesystems=['gluster']) + image_id = 1 # doesn't matter + client = MyGlanceStubClient() + service = self._create_image_service(client) + self.flags(group='image_file_url:gluster', id=file_system_id) + self.flags(group='image_file_url:gluster', mountpoint=nova_mount) + + service.download(self.context, image_id) + self.assertTrue(self.copy_called) + + def test_download_module_file_bad_module(self): + _, data_filename = self._get_tempfile() + file_url = 'applesauce://%s' % data_filename + data_from_client = "someData" + + class MyGlanceStubClient(glance_stubs.StubGlanceClient): + def get(self, image_id): + return type('GlanceLocations', (object,), + {'locations': [{'url': file_url, + 'metadata': {}}]}) + + def data(self, image_id): + return data_from_client + + self.flags(allowed_direct_url_schemes=['applesauce']) + + self.mox.StubOutWithMock(shutil, 'copyfileobj') + self.flags(allowed_direct_url_schemes=['file']) + image_id = 1 # doesn't matter + client = MyGlanceStubClient() + service = self._create_image_service(client) + + # by not calling copyfileobj in the file download module we verify + # that the requirements were not met for its use + self.mox.ReplayAll() + new_data = service.download(self.context, image_id) + self.mox.VerifyAll() + self.assertEqual(data_from_client, new_data) + def test_client_forbidden_converts_to_imagenotauthed(self): class MyGlanceStubClient(glance_stubs.StubGlanceClient): """A client that raises a Forbidden exception.""" diff --git a/setup.cfg b/setup.cfg index f4e7c680324a..c9454a861cf2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,8 @@ packages = nova [entry_points] +nova.image.download.modules = + file = nova.image.download.file console_scripts = nova-all = nova.cmd.all:main nova-api = nova.cmd.api:main