From bf3c74563bdb6e0019db2af14c3df629d429f56b Mon Sep 17 00:00:00 2001 From: Mate Lakat Date: Wed, 28 Aug 2013 15:32:28 +0100 Subject: [PATCH] xenapi: support raw tgz image download Support the download of raw tgz images. The image's size is retrieved by reading the first tarinfo from the stream. related to blueprint xenapi-supported-image-import-export Change-Id: Ibb7a54261b50f73100f4ab4acd59d8e7cd2901a0 --- nova/tests/virt/xenapi/image/test_utils.py | 195 +++++++++++++++++++++ nova/virt/xenapi/image/utils.py | 61 +++++++ nova/virt/xenapi/vm_utils.py | 5 +- 3 files changed, 260 insertions(+), 1 deletion(-) diff --git a/nova/tests/virt/xenapi/image/test_utils.py b/nova/tests/virt/xenapi/image/test_utils.py index 1e7aad16c244..725f6e655997 100644 --- a/nova/tests/virt/xenapi/image/test_utils.py +++ b/nova/tests/virt/xenapi/image/test_utils.py @@ -15,6 +15,8 @@ # License for the specific language governing permissions and limitations # under the License. +import tarfile + from nova.image import glance from nova import test from nova.virt.xenapi.image import utils @@ -70,6 +72,33 @@ class GlanceImageTestCase(test.TestCase): image = self._get_image() self.assertEquals('result', image.download_to('fobj')) + def test_is_raw_tgz_empty_meta(self): + self._stub_out_glance_services() + self.mox.ReplayAll() + + image = self._get_image() + image._cached_meta = {} + + self.assertEquals(False, image.is_raw_tgz()) + + def test_is_raw_tgz_for_raw_tgz(self): + self._stub_out_glance_services() + self.mox.ReplayAll() + + image = self._get_image() + image._cached_meta = {'disk_format': 'raw', 'container_format': 'tgz'} + + self.assertEquals(True, image.is_raw_tgz()) + + def test_data(self): + image_service = self._stub_out_glance_services() + image_service.download('context', 'id').AndReturn('data') + self.mox.ReplayAll() + + image = self._get_image() + + self.assertEquals('data', image.data()) + class RawImageTestCase(test.TestCase): def test_get_size(self): @@ -87,3 +116,169 @@ class RawImageTestCase(test.TestCase): self.mox.ReplayAll() self.assertEquals('result', raw_image.stream_to('file')) + + +class TestIterableBasedFile(test.TestCase): + def test_constructor(self): + class FakeIterable(object): + def __iter__(_self): + return 'iterator' + + the_file = utils.IterableToFileAdapter(FakeIterable()) + + self.assertEquals('iterator', the_file.iterator) + + def test_read_one_character(self): + the_file = utils.IterableToFileAdapter([ + 'chunk1', 'chunk2' + ]) + + self.assertEquals('c', the_file.read(1)) + + def test_read_stores_remaining_characters(self): + the_file = utils.IterableToFileAdapter([ + 'chunk1', 'chunk2' + ]) + + the_file.read(1) + + self.assertEquals('hunk1', the_file.remaining_data) + + def test_read_remaining_characters(self): + the_file = utils.IterableToFileAdapter([ + 'chunk1', 'chunk2' + ]) + + self.assertEquals('c', the_file.read(1)) + self.assertEquals('h', the_file.read(1)) + + def test_read_reached_end_of_file(self): + the_file = utils.IterableToFileAdapter([ + 'chunk1', 'chunk2' + ]) + + self.assertEquals('chunk1', the_file.read(100)) + self.assertEquals('chunk2', the_file.read(100)) + self.assertEquals('', the_file.read(100)) + + def test_empty_chunks(self): + the_file = utils.IterableToFileAdapter([ + '', '', 'chunk2' + ]) + + self.assertEquals('chunk2', the_file.read(100)) + + +class RawTGZTestCase(test.TestCase): + def test_as_tarfile(self): + image = utils.RawTGZImage(None) + self.mox.StubOutWithMock(image, '_as_file') + self.mox.StubOutWithMock(utils.tarfile, 'open') + + image._as_file().AndReturn('the_file') + utils.tarfile.open(mode='r|gz', fileobj='the_file').AndReturn('tf') + + self.mox.ReplayAll() + + result = image._as_tarfile() + self.assertEquals('tf', result) + + def test_as_file(self): + self.mox.StubOutWithMock(utils, 'IterableToFileAdapter') + glance_image = self.mox.CreateMock(utils.GlanceImage) + image = utils.RawTGZImage(glance_image) + glance_image.data().AndReturn('iterable-data') + utils.IterableToFileAdapter('iterable-data').AndReturn('data-as-file') + + self.mox.ReplayAll() + + result = image._as_file() + + self.assertEquals('data-as-file', result) + + def test_get_size(self): + tar_file = self.mox.CreateMock(tarfile.TarFile) + tar_info = self.mox.CreateMock(tarfile.TarInfo) + + image = utils.RawTGZImage(None) + + self.mox.StubOutWithMock(image, '_as_tarfile') + + image._as_tarfile().AndReturn(tar_file) + tar_file.next().AndReturn(tar_info) + tar_info.size = 124 + + self.mox.ReplayAll() + + result = image.get_size() + + self.assertEquals(124, result) + self.assertEquals(image._tar_info, tar_info) + self.assertEquals(image._tar_file, tar_file) + + def test_get_size_called_twice(self): + tar_file = self.mox.CreateMock(tarfile.TarFile) + tar_info = self.mox.CreateMock(tarfile.TarInfo) + + image = utils.RawTGZImage(None) + + self.mox.StubOutWithMock(image, '_as_tarfile') + + image._as_tarfile().AndReturn(tar_file) + tar_file.next().AndReturn(tar_info) + tar_info.size = 124 + + self.mox.ReplayAll() + + image.get_size() + result = image.get_size() + + self.assertEquals(124, result) + self.assertEquals(image._tar_info, tar_info) + self.assertEquals(image._tar_file, tar_file) + + def test_stream_to_without_size_retrieved(self): + source_tar = self.mox.CreateMock(tarfile.TarFile) + first_tarinfo = self.mox.CreateMock(tarfile.TarInfo) + target_file = self.mox.CreateMock(file) + source_file = self.mox.CreateMock(file) + + image = utils.RawTGZImage(None) + image._image_service_and_image_id = ('service', 'id') + + self.mox.StubOutWithMock(image, '_as_tarfile', source_tar) + self.mox.StubOutWithMock(utils.shutil, 'copyfileobj') + + image._as_tarfile().AndReturn(source_tar) + source_tar.next().AndReturn(first_tarinfo) + source_tar.extractfile(first_tarinfo).AndReturn(source_file) + utils.shutil.copyfileobj(source_file, target_file) + source_tar.close() + + self.mox.ReplayAll() + + image.stream_to(target_file) + + def test_stream_to_with_size_retrieved(self): + source_tar = self.mox.CreateMock(tarfile.TarFile) + first_tarinfo = self.mox.CreateMock(tarfile.TarInfo) + target_file = self.mox.CreateMock(file) + source_file = self.mox.CreateMock(file) + first_tarinfo.size = 124 + + image = utils.RawTGZImage(None) + image._image_service_and_image_id = ('service', 'id') + + self.mox.StubOutWithMock(image, '_as_tarfile', source_tar) + self.mox.StubOutWithMock(utils.shutil, 'copyfileobj') + + image._as_tarfile().AndReturn(source_tar) + source_tar.next().AndReturn(first_tarinfo) + source_tar.extractfile(first_tarinfo).AndReturn(source_file) + utils.shutil.copyfileobj(source_file, target_file) + source_tar.close() + + self.mox.ReplayAll() + + image.get_size() + image.stream_to(target_file) diff --git a/nova/virt/xenapi/image/utils.py b/nova/virt/xenapi/image/utils.py index 32781b8cdaa8..5bb994c26ed8 100644 --- a/nova/virt/xenapi/image/utils.py +++ b/nova/virt/xenapi/image/utils.py @@ -15,6 +15,9 @@ # License for the specific language governing permissions and limitations # under the License. +import shutil +import tarfile + from nova.image import glance @@ -36,6 +39,13 @@ class GlanceImage(object): return self._image_service.download( self._context, self._image_id, fileobj) + def is_raw_tgz(self): + return ['raw', 'tgz'] == [ + self.meta.get(key) for key in ('disk_format', 'container_format')] + + def data(self): + return self._image_service.download(self._context, self._image_id) + class RawImage(object): def __init__(self, glance_image): @@ -46,3 +56,54 @@ class RawImage(object): def stream_to(self, fileobj): return self.glance_image.download_to(fileobj) + + +class IterableToFileAdapter(object): + """A degenerate file-like so that an iterable could be read like a file. + + As Glance client returns an iterable, but tarfile requires a file like, + this is the adapter between the two. This allows tarfile to access the + glance stream. + """ + + def __init__(self, iterable): + self.iterator = iterable.__iter__() + self.remaining_data = '' + + def read(self, size): + chunk = self.remaining_data + try: + while not chunk: + chunk = self.iterator.next() + except StopIteration: + return '' + return_value = chunk[0:size] + self.remaining_data = chunk[size:] + return return_value + + +class RawTGZImage(object): + def __init__(self, glance_image): + self.glance_image = glance_image + self._tar_info = None + self._tar_file = None + + def _as_file(self): + return IterableToFileAdapter(self.glance_image.data()) + + def _as_tarfile(self): + return tarfile.open(mode='r|gz', fileobj=self._as_file()) + + def get_size(self): + if self._tar_file is None: + self._tar_file = self._as_tarfile() + self._tar_info = self._tar_file.next() + return self._tar_info.size + + def stream_to(self, target_file): + if self._tar_file is None: + self._tar_file = self._as_tarfile() + self._tar_info = self._tar_file.next() + source_file = self._tar_file.extractfile(self._tar_info) + shutil.copyfileobj(source_file, target_file) + self._tar_file.close() diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index c68cef7d5a33..2bfdd1fe65df 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -1297,7 +1297,10 @@ def _fetch_disk_image(context, session, instance, name_label, image_id, sr_ref = safe_find_sr(session) glance_image = image_utils.GlanceImage(context, image_id) - image = image_utils.RawImage(glance_image) + if glance_image.is_raw_tgz(): + image = image_utils.RawTGZImage(glance_image) + else: + image = image_utils.RawImage(glance_image) virtual_size = image.get_size() vdi_size = virtual_size