Merge "xenapi: support raw tgz image download"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
b880ed4491
@ -15,6 +15,8 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import tarfile
|
||||
|
||||
from nova.image import glance
|
||||
from nova import test
|
||||
from nova.virt.xenapi.image import utils
|
||||
@ -70,6 +72,33 @@ class GlanceImageTestCase(test.TestCase):
|
||||
image = self._get_image()
|
||||
self.assertEquals('result', image.download_to('fobj'))
|
||||
|
||||
def test_is_raw_tgz_empty_meta(self):
|
||||
self._stub_out_glance_services()
|
||||
self.mox.ReplayAll()
|
||||
|
||||
image = self._get_image()
|
||||
image._cached_meta = {}
|
||||
|
||||
self.assertEquals(False, image.is_raw_tgz())
|
||||
|
||||
def test_is_raw_tgz_for_raw_tgz(self):
|
||||
self._stub_out_glance_services()
|
||||
self.mox.ReplayAll()
|
||||
|
||||
image = self._get_image()
|
||||
image._cached_meta = {'disk_format': 'raw', 'container_format': 'tgz'}
|
||||
|
||||
self.assertEquals(True, image.is_raw_tgz())
|
||||
|
||||
def test_data(self):
|
||||
image_service = self._stub_out_glance_services()
|
||||
image_service.download('context', 'id').AndReturn('data')
|
||||
self.mox.ReplayAll()
|
||||
|
||||
image = self._get_image()
|
||||
|
||||
self.assertEquals('data', image.data())
|
||||
|
||||
|
||||
class RawImageTestCase(test.TestCase):
|
||||
def test_get_size(self):
|
||||
@ -87,3 +116,169 @@ class RawImageTestCase(test.TestCase):
|
||||
self.mox.ReplayAll()
|
||||
|
||||
self.assertEquals('result', raw_image.stream_to('file'))
|
||||
|
||||
|
||||
class TestIterableBasedFile(test.TestCase):
|
||||
def test_constructor(self):
|
||||
class FakeIterable(object):
|
||||
def __iter__(_self):
|
||||
return 'iterator'
|
||||
|
||||
the_file = utils.IterableToFileAdapter(FakeIterable())
|
||||
|
||||
self.assertEquals('iterator', the_file.iterator)
|
||||
|
||||
def test_read_one_character(self):
|
||||
the_file = utils.IterableToFileAdapter([
|
||||
'chunk1', 'chunk2'
|
||||
])
|
||||
|
||||
self.assertEquals('c', the_file.read(1))
|
||||
|
||||
def test_read_stores_remaining_characters(self):
|
||||
the_file = utils.IterableToFileAdapter([
|
||||
'chunk1', 'chunk2'
|
||||
])
|
||||
|
||||
the_file.read(1)
|
||||
|
||||
self.assertEquals('hunk1', the_file.remaining_data)
|
||||
|
||||
def test_read_remaining_characters(self):
|
||||
the_file = utils.IterableToFileAdapter([
|
||||
'chunk1', 'chunk2'
|
||||
])
|
||||
|
||||
self.assertEquals('c', the_file.read(1))
|
||||
self.assertEquals('h', the_file.read(1))
|
||||
|
||||
def test_read_reached_end_of_file(self):
|
||||
the_file = utils.IterableToFileAdapter([
|
||||
'chunk1', 'chunk2'
|
||||
])
|
||||
|
||||
self.assertEquals('chunk1', the_file.read(100))
|
||||
self.assertEquals('chunk2', the_file.read(100))
|
||||
self.assertEquals('', the_file.read(100))
|
||||
|
||||
def test_empty_chunks(self):
|
||||
the_file = utils.IterableToFileAdapter([
|
||||
'', '', 'chunk2'
|
||||
])
|
||||
|
||||
self.assertEquals('chunk2', the_file.read(100))
|
||||
|
||||
|
||||
class RawTGZTestCase(test.TestCase):
|
||||
def test_as_tarfile(self):
|
||||
image = utils.RawTGZImage(None)
|
||||
self.mox.StubOutWithMock(image, '_as_file')
|
||||
self.mox.StubOutWithMock(utils.tarfile, 'open')
|
||||
|
||||
image._as_file().AndReturn('the_file')
|
||||
utils.tarfile.open(mode='r|gz', fileobj='the_file').AndReturn('tf')
|
||||
|
||||
self.mox.ReplayAll()
|
||||
|
||||
result = image._as_tarfile()
|
||||
self.assertEquals('tf', result)
|
||||
|
||||
def test_as_file(self):
|
||||
self.mox.StubOutWithMock(utils, 'IterableToFileAdapter')
|
||||
glance_image = self.mox.CreateMock(utils.GlanceImage)
|
||||
image = utils.RawTGZImage(glance_image)
|
||||
glance_image.data().AndReturn('iterable-data')
|
||||
utils.IterableToFileAdapter('iterable-data').AndReturn('data-as-file')
|
||||
|
||||
self.mox.ReplayAll()
|
||||
|
||||
result = image._as_file()
|
||||
|
||||
self.assertEquals('data-as-file', result)
|
||||
|
||||
def test_get_size(self):
|
||||
tar_file = self.mox.CreateMock(tarfile.TarFile)
|
||||
tar_info = self.mox.CreateMock(tarfile.TarInfo)
|
||||
|
||||
image = utils.RawTGZImage(None)
|
||||
|
||||
self.mox.StubOutWithMock(image, '_as_tarfile')
|
||||
|
||||
image._as_tarfile().AndReturn(tar_file)
|
||||
tar_file.next().AndReturn(tar_info)
|
||||
tar_info.size = 124
|
||||
|
||||
self.mox.ReplayAll()
|
||||
|
||||
result = image.get_size()
|
||||
|
||||
self.assertEquals(124, result)
|
||||
self.assertEquals(image._tar_info, tar_info)
|
||||
self.assertEquals(image._tar_file, tar_file)
|
||||
|
||||
def test_get_size_called_twice(self):
|
||||
tar_file = self.mox.CreateMock(tarfile.TarFile)
|
||||
tar_info = self.mox.CreateMock(tarfile.TarInfo)
|
||||
|
||||
image = utils.RawTGZImage(None)
|
||||
|
||||
self.mox.StubOutWithMock(image, '_as_tarfile')
|
||||
|
||||
image._as_tarfile().AndReturn(tar_file)
|
||||
tar_file.next().AndReturn(tar_info)
|
||||
tar_info.size = 124
|
||||
|
||||
self.mox.ReplayAll()
|
||||
|
||||
image.get_size()
|
||||
result = image.get_size()
|
||||
|
||||
self.assertEquals(124, result)
|
||||
self.assertEquals(image._tar_info, tar_info)
|
||||
self.assertEquals(image._tar_file, tar_file)
|
||||
|
||||
def test_stream_to_without_size_retrieved(self):
|
||||
source_tar = self.mox.CreateMock(tarfile.TarFile)
|
||||
first_tarinfo = self.mox.CreateMock(tarfile.TarInfo)
|
||||
target_file = self.mox.CreateMock(file)
|
||||
source_file = self.mox.CreateMock(file)
|
||||
|
||||
image = utils.RawTGZImage(None)
|
||||
image._image_service_and_image_id = ('service', 'id')
|
||||
|
||||
self.mox.StubOutWithMock(image, '_as_tarfile', source_tar)
|
||||
self.mox.StubOutWithMock(utils.shutil, 'copyfileobj')
|
||||
|
||||
image._as_tarfile().AndReturn(source_tar)
|
||||
source_tar.next().AndReturn(first_tarinfo)
|
||||
source_tar.extractfile(first_tarinfo).AndReturn(source_file)
|
||||
utils.shutil.copyfileobj(source_file, target_file)
|
||||
source_tar.close()
|
||||
|
||||
self.mox.ReplayAll()
|
||||
|
||||
image.stream_to(target_file)
|
||||
|
||||
def test_stream_to_with_size_retrieved(self):
|
||||
source_tar = self.mox.CreateMock(tarfile.TarFile)
|
||||
first_tarinfo = self.mox.CreateMock(tarfile.TarInfo)
|
||||
target_file = self.mox.CreateMock(file)
|
||||
source_file = self.mox.CreateMock(file)
|
||||
first_tarinfo.size = 124
|
||||
|
||||
image = utils.RawTGZImage(None)
|
||||
image._image_service_and_image_id = ('service', 'id')
|
||||
|
||||
self.mox.StubOutWithMock(image, '_as_tarfile', source_tar)
|
||||
self.mox.StubOutWithMock(utils.shutil, 'copyfileobj')
|
||||
|
||||
image._as_tarfile().AndReturn(source_tar)
|
||||
source_tar.next().AndReturn(first_tarinfo)
|
||||
source_tar.extractfile(first_tarinfo).AndReturn(source_file)
|
||||
utils.shutil.copyfileobj(source_file, target_file)
|
||||
source_tar.close()
|
||||
|
||||
self.mox.ReplayAll()
|
||||
|
||||
image.get_size()
|
||||
image.stream_to(target_file)
|
||||
|
||||
@ -15,6 +15,9 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import shutil
|
||||
import tarfile
|
||||
|
||||
from nova.image import glance
|
||||
|
||||
|
||||
@ -36,6 +39,13 @@ class GlanceImage(object):
|
||||
return self._image_service.download(
|
||||
self._context, self._image_id, fileobj)
|
||||
|
||||
def is_raw_tgz(self):
|
||||
return ['raw', 'tgz'] == [
|
||||
self.meta.get(key) for key in ('disk_format', 'container_format')]
|
||||
|
||||
def data(self):
|
||||
return self._image_service.download(self._context, self._image_id)
|
||||
|
||||
|
||||
class RawImage(object):
|
||||
def __init__(self, glance_image):
|
||||
@ -46,3 +56,54 @@ class RawImage(object):
|
||||
|
||||
def stream_to(self, fileobj):
|
||||
return self.glance_image.download_to(fileobj)
|
||||
|
||||
|
||||
class IterableToFileAdapter(object):
|
||||
"""A degenerate file-like so that an iterable could be read like a file.
|
||||
|
||||
As Glance client returns an iterable, but tarfile requires a file like,
|
||||
this is the adapter between the two. This allows tarfile to access the
|
||||
glance stream.
|
||||
"""
|
||||
|
||||
def __init__(self, iterable):
|
||||
self.iterator = iterable.__iter__()
|
||||
self.remaining_data = ''
|
||||
|
||||
def read(self, size):
|
||||
chunk = self.remaining_data
|
||||
try:
|
||||
while not chunk:
|
||||
chunk = self.iterator.next()
|
||||
except StopIteration:
|
||||
return ''
|
||||
return_value = chunk[0:size]
|
||||
self.remaining_data = chunk[size:]
|
||||
return return_value
|
||||
|
||||
|
||||
class RawTGZImage(object):
|
||||
def __init__(self, glance_image):
|
||||
self.glance_image = glance_image
|
||||
self._tar_info = None
|
||||
self._tar_file = None
|
||||
|
||||
def _as_file(self):
|
||||
return IterableToFileAdapter(self.glance_image.data())
|
||||
|
||||
def _as_tarfile(self):
|
||||
return tarfile.open(mode='r|gz', fileobj=self._as_file())
|
||||
|
||||
def get_size(self):
|
||||
if self._tar_file is None:
|
||||
self._tar_file = self._as_tarfile()
|
||||
self._tar_info = self._tar_file.next()
|
||||
return self._tar_info.size
|
||||
|
||||
def stream_to(self, target_file):
|
||||
if self._tar_file is None:
|
||||
self._tar_file = self._as_tarfile()
|
||||
self._tar_info = self._tar_file.next()
|
||||
source_file = self._tar_file.extractfile(self._tar_info)
|
||||
shutil.copyfileobj(source_file, target_file)
|
||||
self._tar_file.close()
|
||||
|
||||
@ -1303,6 +1303,9 @@ def _fetch_disk_image(context, session, instance, name_label, image_id,
|
||||
sr_ref = safe_find_sr(session)
|
||||
|
||||
glance_image = image_utils.GlanceImage(context, image_id)
|
||||
if glance_image.is_raw_tgz():
|
||||
image = image_utils.RawTGZImage(glance_image)
|
||||
else:
|
||||
image = image_utils.RawImage(glance_image)
|
||||
|
||||
virtual_size = image.get_size()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user