Add ability to download Glance images into the libvirt image cache via RBD

This change allows compute hosts to quickly download and cache images on the
local compute host directly from Ceph rather than slow dowloads from the
Glance API.

New '[glance]/enable_rbd_download' option is introduced to enable this
behavior. This is slight change compared to the original idea described in
the relevant blueprint where it was discussed to use (now obsolete)
'[glance]/allowed_direct_url_schemes' option.

Additionally, when an image signature verification is requested, it should
be done also for the image fetched by the new download handler. This was
completely missing so far.

New '[glance]/rbd_{user,pool,ceph_conf,connect_timeout}' configurables
are introduced to allow operators to configure access to the cluster
hosting Glance without the need to use the existing '[libvirt]'
specific configurables. nova.storage.rbd_utils.RBDDriver has also been
modified to accept these but continues to default to the '[libvirt]'
specific configurables for now.

Change-Id: I3032bbe6bd2d6acc9ba0f0cac4d00ed4b4464ceb
Implements: blueprint nova-image-download-via-rbd
This commit is contained in:
Curt Moore 2018-06-11 10:08:57 -05:00 committed by Jiří Suchomel
parent b5d4804346
commit 61aeb1adbc
5 changed files with 319 additions and 13 deletions

View File

@ -148,9 +148,64 @@ Related options:
* The value of this option may be used if both verify_glance_signatures and
enable_certificate_validation are enabled.
"""),
cfg.BoolOpt('enable_rbd_download',
default=False,
help="""
Enable download of Glance images directly via RBD.
Allow compute hosts to quickly download and cache images localy directly
from Ceph rather than slow dowloads from the Glance API. This can
reduce download time for images in the ten to hundreds of GBs from tens of
minutes to tens of seconds, but requires a Ceph-based deployment and access
from the compute nodes to Ceph.
Related options:
* ``[glance] rbd_user``
* ``[glance] rbd_connect_timeout``
* ``[glance] rbd_pool``
* ``[glance] rbd_ceph_conf``
"""),
cfg.StrOpt('rbd_user',
default='',
help="""
The RADOS client name for accessing Glance images stored as rbd volumes.
Related options:
* This option is only used if ``[glance] enable_rbd_download`` is set to True.
"""),
cfg.IntOpt('rbd_connect_timeout',
default=5,
help="""
The RADOS client timeout in seconds when initially connecting to the cluster.
Related options:
* This option is only used if ``[glance] enable_rbd_download`` is set to True.
"""),
cfg.StrOpt('rbd_pool',
default='',
help="""
The RADOS pool in which the Glance images are stored as rbd volumes.
Related options:
* This option is only used if ``[glance] enable_rbd_download`` is set to True.
"""),
cfg.StrOpt('rbd_ceph_conf',
default='',
help="""
Path to the ceph configuration file to use.
Related options:
* This option is only used if ``[glance] enable_rbd_download`` is set to True.
"""),
cfg.BoolOpt('debug',
default=False,
help='Enable or disable debug logging with glanceclient.')
default=False,
help='Enable or disable debug logging with glanceclient.')
]
deprecated_ksa_opts = {

View File

@ -24,12 +24,14 @@ import re
import stat
import sys
import time
import urllib.parse as urlparse
import cryptography
from cursive import certificate_utils
from cursive import exception as cursive_exception
from cursive import signature_utils
import glanceclient
from glanceclient.common import utils as glance_utils
import glanceclient.exc
from glanceclient.v2 import schemas
from keystoneauth1 import loading as ks_loading
@ -39,7 +41,6 @@ from oslo_utils import excutils
from oslo_utils import timeutils
import six
from six.moves import range
import six.moves.urllib.parse as urlparse
import nova.conf
from nova import exception
@ -221,6 +222,51 @@ class GlanceImageServiceV2(object):
# to be added here.
self._download_handlers = {}
if CONF.glance.enable_rbd_download:
self._download_handlers['rbd'] = self.rbd_download
def rbd_download(self, context, url_parts, dst_path, metadata=None):
"""Use an explicit rbd call to download an image.
:param context: The `nova.context.RequestContext` object for the
request
:param url_parts: Parts of URL pointing to the image location
:param dst_path: Filepath to transfer the image file to.
:param metadata: Image location metadata (currently unused)
"""
# avoid circular import
from nova.storage import rbd_utils
try:
# Parse the RBD URL from url_parts, it should consist of 4
# sections and be in the format of:
# <cluster_uuid>/<pool_name>/<image_uuid>/<snapshot_name>
url_path = str(urlparse.unquote(url_parts.path))
cluster_uuid, pool_name, image_uuid, snapshot_name = (
url_path.split('/'))
except ValueError as e:
msg = f"Invalid RBD URL format: {e}"
LOG.error(msg)
raise nova.exception.InvalidParameterValue(msg)
rbd_driver = rbd_utils.RBDDriver(
user=CONF.glance.rbd_user,
pool=CONF.glance.rbd_pool,
ceph_conf=CONF.glance.rbd_ceph_conf,
connect_timeout=CONF.glance.rbd_connect_timeout)
try:
LOG.debug("Attempting to export RBD image: "
"[pool_name: %s] [image_uuid: %s] "
"[snapshot_name: %s] [dst_path: %s]",
pool_name, image_uuid, snapshot_name, dst_path)
rbd_driver.export_image(dst_path, image_uuid,
snapshot_name, pool_name)
except Exception as e:
LOG.error("Error during RBD image export: %s", e)
raise nova.exception.CouldNotFetchImage(image_id=image_uuid)
def show(self, context, image_id, include_locations=False,
show_deleted=True):
"""Returns a dict with image data for the given opaque image id.
@ -299,7 +345,13 @@ class GlanceImageServiceV2(object):
def download(self, context, image_id, data=None, dst_path=None,
trusted_certs=None):
"""Calls out to Glance for data and writes data."""
if CONF.glance.allowed_direct_url_schemes and dst_path is not None:
# First, check if image could be directly downloaded by special handler
# TODO(stephenfin): Remove check for 'allowed_direct_url_schemes' when
# we clean up tests since it's not used elsewhere
if ((CONF.glance.allowed_direct_url_schemes or
self._download_handlers) and dst_path is not None
):
image = self.show(context, image_id, include_locations=True)
for entry in image.get('locations', []):
loc_url = entry['url']
@ -310,10 +362,21 @@ class GlanceImageServiceV2(object):
try:
xfer_method(context, o, dst_path, loc_meta)
LOG.info("Successfully transferred using %s", o.scheme)
# Load chunks from the downloaded image file
# for verification (if required)
with open(dst_path, 'rb') as fh:
downloaded_length = os.path.getsize(dst_path)
image_chunks = glance_utils.IterableWithLength(fh,
downloaded_length)
self._verify_and_write(context, image_id,
trusted_certs, image_chunks, None, None)
return
except Exception:
LOG.exception("Download image error")
# By default (or if direct download has failed), use glance client call
# to fetch the image and fill image_chunks
try:
image_chunks = self._client.call(
context, 2, 'data', args=(image_id,))

View File

@ -122,14 +122,16 @@ class RADOSClient(object):
class RBDDriver(object):
def __init__(self):
def __init__(self, pool=None, user=None, ceph_conf=None,
connect_timeout=None):
if rbd is None:
raise RuntimeError(_('rbd python libraries not found'))
self.pool = CONF.libvirt.images_rbd_pool
self.rbd_user = CONF.libvirt.rbd_user
self.rbd_connect_timeout = CONF.libvirt.rbd_connect_timeout
self.ceph_conf = CONF.libvirt.images_rbd_ceph_conf
self.pool = pool or CONF.libvirt.images_rbd_pool
self.rbd_user = user or CONF.libvirt.rbd_user
self.rbd_connect_timeout = (
connect_timeout or CONF.libvirt.rbd_connect_timeout)
self.ceph_conf = ceph_conf or CONF.libvirt.images_rbd_ceph_conf
def _connect_to_rados(self, pool=None):
client = rados.Rados(rados_id=self.rbd_user,
@ -335,6 +337,25 @@ class RBDDriver(object):
args += self.ceph_args()
processutils.execute('rbd', 'import', *args)
def export_image(self, base, name, snap, pool=None):
"""Export RBD volume to image file.
Uses the command line export to export rbd volume snapshot to
local image file.
:base: Path to image file
:name: Name of RBD volume
:snap: Name of RBD snapshot
:pool: Name of RBD pool
"""
if pool is None:
pool = self.pool
args = ['--pool', pool, '--image', name, '--path', base,
'--snap', snap]
args += self.ceph_args()
processutils.execute('rbd', 'export', *args)
def _destroy_volume(self, client, volume, pool=None):
"""Destroy an RBD volume, retrying as needed.
"""

View File

@ -16,6 +16,7 @@
import copy
import datetime
import urllib.parse as urlparse
import cryptography
from cursive import exception as cursive_exception
@ -36,6 +37,7 @@ from nova import exception
from nova.image import glance
from nova import objects
from nova import service_auth
from nova.storage import rbd_utils
from nova import test
@ -686,9 +688,14 @@ class TestDownloadNoDirectUri(test.NoDBTestCase):
with testtools.ExpectedException(exception.ImageUnacceptable):
service.download(ctx, mock.sentinel.image_id)
# TODO(stephenfin): Drop this test since it's not possible to run in
# production
@mock.patch('os.path.getsize', return_value=1)
@mock.patch.object(six.moves.builtins, 'open')
@mock.patch('nova.image.glance.GlanceImageServiceV2._get_transfer_method')
@mock.patch('nova.image.glance.GlanceImageServiceV2.show')
def test_download_direct_file_uri_v2(self, show_mock, get_tran_mock):
def test_download_direct_file_uri_v2(
self, show_mock, get_tran_mock, open_mock, getsize_mock):
self.flags(allowed_direct_url_schemes=['file'], group='glance')
show_mock.return_value = {
'locations': [
@ -702,6 +709,8 @@ class TestDownloadNoDirectUri(test.NoDBTestCase):
get_tran_mock.return_value = tran_mod
client = mock.MagicMock()
ctx = mock.sentinel.ctx
writer = mock.MagicMock()
open_mock.return_value = writer
service = glance.GlanceImageServiceV2(client)
res = service.download(ctx, mock.sentinel.image_id,
dst_path=mock.sentinel.dst_path)
@ -716,6 +725,76 @@ class TestDownloadNoDirectUri(test.NoDBTestCase):
mock.sentinel.dst_path,
mock.sentinel.loc_meta)
@mock.patch('glanceclient.common.utils.IterableWithLength')
@mock.patch('os.path.getsize', return_value=1)
@mock.patch.object(six.moves.builtins, 'open')
@mock.patch('nova.image.glance.LOG')
@mock.patch('nova.image.glance.GlanceImageServiceV2._get_verifier')
@mock.patch('nova.image.glance.GlanceImageServiceV2._get_transfer_method')
@mock.patch('nova.image.glance.GlanceImageServiceV2.show')
def test_download_direct_rbd_uri_v2(
self, show_mock, get_tran_mock, get_verifier_mock, log_mock,
open_mock, getsize_mock, iterable_with_length_mock):
self.flags(enable_rbd_download=True, group='glance')
show_mock.return_value = {
'locations': [
{
'url': 'rbd://cluster_uuid/pool_name/image_uuid/snapshot',
'metadata': mock.sentinel.loc_meta
}
]
}
tran_mod = mock.MagicMock()
get_tran_mock.return_value = tran_mod
client = mock.MagicMock()
ctx = mock.sentinel.ctx
writer = mock.MagicMock()
open_mock.return_value = writer
iterable_with_length_mock.return_value = ["rbd1", "rbd2"]
service = glance.GlanceImageServiceV2(client)
verifier = mock.MagicMock()
get_verifier_mock.return_value = verifier
res = service.download(ctx, mock.sentinel.image_id,
dst_path=mock.sentinel.dst_path,
trusted_certs=mock.sentinel.trusted_certs)
self.assertIsNone(res)
show_mock.assert_called_once_with(ctx,
mock.sentinel.image_id,
include_locations=True)
tran_mod.assert_called_once_with(ctx, mock.ANY,
mock.sentinel.dst_path,
mock.sentinel.loc_meta)
open_mock.assert_called_once_with(mock.sentinel.dst_path, 'rb')
get_tran_mock.assert_called_once_with('rbd')
# no client call, chunks were read right after xfer_mod.download:
client.call.assert_not_called()
# verifier called with the value we got from rbd download
verifier.update.assert_has_calls(
[
mock.call("rbd1"),
mock.call("rbd2")
]
)
verifier.verify.assert_called()
log_mock.info.assert_has_calls(
[
mock.call('Successfully transferred using %s', 'rbd'),
mock.call(
'Image signature verification succeeded for image %s',
mock.sentinel.image_id)
]
)
# not opened for writing (already written)
self.assertFalse(open_mock(mock.sentinel.dst_path, 'rw').called)
# write not called (written by rbd download)
writer.write.assert_not_called()
@mock.patch('nova.image.glance.GlanceImageServiceV2._get_transfer_method')
@mock.patch('nova.image.glance.GlanceImageServiceV2.show')
@mock.patch('nova.image.glance.GlanceImageServiceV2._safe_fsync')
@ -1249,6 +1328,60 @@ class TestIsImageAvailable(test.NoDBTestCase):
self.assertTrue(res)
class TestRBDDownload(test.NoDBTestCase):
def setUp(self):
super(TestRBDDownload, self).setUp()
loc_url = "rbd://ce2d1ace/images/b86d6d06-faac/snap"
self.url_parts = urlparse.urlparse(loc_url)
self.image_uuid = "b86d6d06-faac"
self.pool_name = "images"
self.snapshot_name = "snap"
@mock.patch.object(rbd_utils.RBDDriver, 'export_image')
@mock.patch.object(rbd_utils, 'rbd')
def test_rbd_download_success(self, mock_rbd, mock_export_image):
client = mock.MagicMock()
ctx = mock.sentinel.ctx
service = glance.GlanceImageServiceV2(client)
service.rbd_download(ctx, self.url_parts, mock.sentinel.dst_path)
# Assert that we attempt to export using the correct rbd pool, volume
# and snapshot given the provided URL
mock_export_image.assert_called_once_with(mock.sentinel.dst_path,
self.image_uuid,
self.snapshot_name,
self.pool_name)
def test_rbd_download_broken_url(self):
client = mock.MagicMock()
ctx = mock.sentinel.ctx
service = glance.GlanceImageServiceV2(client)
wrong_url = "http://www.example.com"
wrong_url_parts = urlparse.urlparse(wrong_url)
# Assert InvalidParameterValue is raised when we can't parse the URL
self.assertRaises(
exception.InvalidParameterValue, service.rbd_download, ctx,
wrong_url_parts, mock.sentinel.dst_path)
@mock.patch('nova.storage.rbd_utils.RBDDriver.export_image')
@mock.patch.object(rbd_utils, 'rbd')
def test_rbd_download_export_failure(self, mock_rbd, mock_export_image):
client = mock.MagicMock()
ctx = mock.sentinel.ctx
service = glance.GlanceImageServiceV2(client)
mock_export_image.side_effect = Exception
# Assert CouldNotFetchImage is raised when the export fails
self.assertRaisesRegex(
exception.CouldNotFetchImage, self.image_uuid,
service.rbd_download, ctx, self.url_parts, mock.sentinel.dst_path)
class TestShow(test.NoDBTestCase):
"""Tests the show method of the GlanceImageServiceV2."""

View File

@ -109,9 +109,11 @@ class RbdTestCase(test.NoDBTestCase):
self.rbd_pool = 'rbd'
self.rbd_connect_timeout = 5
self.flags(images_rbd_pool=self.rbd_pool, group='libvirt')
self.flags(rbd_connect_timeout=self.rbd_connect_timeout,
group='libvirt')
self.flags(
images_rbd_pool=self.images_rbd_pool,
images_rbd_ceph_conf='/foo/bar.conf',
rbd_connect_timeout=self.rbd_connect_timeout,
rbd_user='foo', group='libvirt')
rados_patcher = mock.patch.object(rbd_utils, 'rados')
self.mock_rados = rados_patcher.start()
@ -657,3 +659,35 @@ class RbdTestCase(test.NoDBTestCase):
ceph_df_not_found = CEPH_DF.replace('rbd', 'vms')
mock_execute.return_value = (ceph_df_not_found, '')
self.assertRaises(exception.NotFound, self.driver.get_pool_info)
@mock.patch('oslo_concurrency.processutils.execute')
def test_export_image(self, mock_execute):
self.driver.rbd_user = 'foo'
self.driver.export_image(mock.sentinel.dst_path,
mock.sentinel.name,
mock.sentinel.snap,
mock.sentinel.pool)
mock_execute.assert_called_once_with(
'rbd', 'export',
'--pool', mock.sentinel.pool,
'--image', mock.sentinel.name,
'--path', mock.sentinel.dst_path,
'--snap', mock.sentinel.snap,
'--id', 'foo',
'--conf', '/foo/bar.conf')
@mock.patch('oslo_concurrency.processutils.execute')
def test_export_image_default_pool(self, mock_execute):
self.driver.export_image(mock.sentinel.dst_path,
mock.sentinel.name,
mock.sentinel.snap)
mock_execute.assert_called_once_with(
'rbd', 'export',
'--pool', self.rbd_pool,
'--image', mock.sentinel.name,
'--path', mock.sentinel.dst_path,
'--snap', mock.sentinel.snap,
'--id', 'foo',
'--conf', '/foo/bar.conf')