x/deb-oslo.vmware
Code Issues Proposed changes

Support download of virtual disk in ova container

Browse Source 
This patch updates download_stream_optimized_image method
in image_transfer to support streamOptimized disks in ova
container.

Change-Id: I693101f8b22eb416c5c0f92957ebb680d3b5bd8b
This commit is contained in:
Vipin Balachandran 2016-03-31 15:05:46 +05:30
parent b10c7575a6
commit 784ad53fc4
6 changed files with 361 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
oslo_vmware/image_transfer.py
View File

@ -19,6 +19,7 @@ Functions and classes for image transfer between ESX/VC & image service.
import errno import errno
import logging import logging
import tarfile
from eventlet import event from eventlet import event
from eventlet import greenthread from eventlet import greenthread
@ -28,6 +29,7 @@ from eventlet import timeout
from oslo_vmware._i18n import _ from oslo_vmware._i18n import _
from oslo_vmware import constants from oslo_vmware import constants
from oslo_vmware import exceptions from oslo_vmware import exceptions
from oslo_vmware import image_util
from oslo_vmware.objects import datastore as ds_obj from oslo_vmware.objects import datastore as ds_obj
from oslo_vmware import rw_handles from oslo_vmware import rw_handles
from oslo_vmware import vim_util from oslo_vmware import vim_util
@ -496,6 +498,19 @@ def download_stream_optimized_data(context, timeout_secs, read_handle,
return write_handle.get_imported_vm() return write_handle.get_imported_vm()
def _get_vmdk_handle(ova_handle):
with tarfile.open(mode="r|", fileobj=ova_handle) as tar:
vmdk_name = None
for tar_info in tar:
if tar_info and tar_info.name.endswith(".ovf"):
vmdk_name = image_util.get_vmdk_name_from_ovf(
tar.extractfile(tar_info))
elif vmdk_name and tar_info.name.startswith(vmdk_name):
# Actual file name is <vmdk_name>.XXXXXXX
return tar.extractfile(tar_info)
def download_stream_optimized_image(context, timeout_secs, image_service, def download_stream_optimized_image(context, timeout_secs, image_service,
image_id, **kwargs): image_id, **kwargs):
"""Download stream optimized image from image service to VMware server. """Download stream optimized image from image service to VMware server.
@ -512,13 +527,24 @@ def download_stream_optimized_image(context, timeout_secs, image_service,
VimSessionOverLoadException, VimConnectionException, VimSessionOverLoadException, VimConnectionException,
ImageTransferException, ValueError ImageTransferException, ValueError
""" """
LOG.debug("Downloading image: %s from image service as a stream " metadata = image_service.show(context, image_id)
"optimized file.", container_format = metadata.get('container_format')
image_id)
LOG.debug("Downloading image: %(id)s (container: %(container)s) from image"
" service as a stream optimized file.",
{'id': image_id,
'container': container_format})
# TODO(vbala) catch specific exceptions raised by download call # TODO(vbala) catch specific exceptions raised by download call
read_iter = image_service.download(context, image_id) read_iter = image_service.download(context, image_id)
read_handle = rw_handles.ImageReadHandle(read_iter) read_handle = rw_handles.ImageReadHandle(read_iter)
if container_format == 'ova':
read_handle = _get_vmdk_handle(read_handle)
if read_handle is None:
raise exceptions.ImageTransferException(
_("No vmdk found in the OVA image %s.") % image_id)
imported_vm = download_stream_optimized_data(context, timeout_secs, imported_vm = download_stream_optimized_data(context, timeout_secs,
read_handle, **kwargs) read_handle, **kwargs)

30
oslo_vmware/image_util.py Normal file
View File

@ -0,0 +1,30 @@
# Copyright (c) 2016 VMware, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from lxml import etree # nosec (bandit bug 1582516)
def _get_vmdk_name_from_ovf(root):
ns_ovf = "{{{0}}}".format(root.nsmap["ovf"])
disk = root.find("./{0}DiskSection/{0}Disk".format(ns_ovf))
file_id = disk.get("{0}fileRef".format(ns_ovf))
f = root.find('./{0}References/{0}File[@{0}id="{1}"]'.format(ns_ovf,
file_id))
return f.get("{0}href".format(ns_ovf))
def get_vmdk_name_from_ovf(ovf_handle):
"""Get the vmdk name from the given ovf descriptor."""
return _get_vmdk_name_from_ovf(etree.parse(ovf_handle).getroot())

136
oslo_vmware/tests/test.ovf Normal file
View File

@ -0,0 +1,136 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--Generated by VMware ESX Server, User: root, UTC time: 2016-03-28T12:06:49.398495Z-->
<Envelope vmw:buildId="build-1331820" xmlns="http://schemas.dmtf.org/ovf/envelope/1" xmlns:cim="http://schemas.dmtf.org/wbem/wscim/1/common" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1" xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:vmw="http://www.vmware.com/schema/ovf" xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<References>
<File ovf:href="test-file1.iso" ovf:id="file1" ovf:size="419840" />
<File ovf:href="test-disk1.vmdk" ovf:id="file2" ovf:size="12046336" />
</References>
<DiskSection>
<Info>Virtual disk information</Info>
<Disk ovf:capacity="1" ovf:capacityAllocationUnits="byte * 2^30" ovf:diskId="vmdisk1" ovf:fileRef="file2" ovf:format="http://www.vmware.com/interfaces/specifications/vmdk.html#streamOptimized" ovf:populatedSize="17498112" />
</DiskSection>
<NetworkSection>
<Info>The list of logical networks</Info>
<Network ovf:name="dvportgroup-81">
<Description>The dvportgroup-81 network</Description>
</Network>
</NetworkSection>
<VirtualSystem ovf:id="test">
<Info>A virtual machine</Info>
<Name>test</Name>
<OperatingSystemSection ovf:id="1" vmw:osType="otherGuest">
<Info>The kind of installed guest operating system</Info>
</OperatingSystemSection>
<VirtualHardwareSection>
<Info>Virtual hardware requirements</Info>
<System>
<vssd:ElementName>Virtual Hardware Family</vssd:ElementName>
<vssd:InstanceID>0</vssd:InstanceID>
<vssd:VirtualSystemIdentifier>test</vssd:VirtualSystemIdentifier>
<vssd:VirtualSystemType>vmx-10</vssd:VirtualSystemType>
</System>
<Item>
<rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
<rasd:Description>Number of Virtual CPUs</rasd:Description>
<rasd:ElementName>1 virtual CPU(s)</rasd:ElementName>
<rasd:InstanceID>1</rasd:InstanceID>
<rasd:ResourceType>3</rasd:ResourceType>
<rasd:VirtualQuantity>1</rasd:VirtualQuantity>
</Item>
<Item>
<rasd:AllocationUnits>byte * 2^20</rasd:AllocationUnits>
<rasd:Description>Memory Size</rasd:Description>
<rasd:ElementName>512MB of memory</rasd:ElementName>
<rasd:InstanceID>2</rasd:InstanceID>
<rasd:ResourceType>4</rasd:ResourceType>
<rasd:VirtualQuantity>512</rasd:VirtualQuantity>
</Item>
<Item>
<rasd:Address>1</rasd:Address>
<rasd:Description>IDE Controller</rasd:Description>
<rasd:ElementName>VirtualIDEController 1</rasd:ElementName>
<rasd:InstanceID>3</rasd:InstanceID>
<rasd:ResourceType>5</rasd:ResourceType>
</Item>
<Item>
<rasd:Address>0</rasd:Address>
<rasd:Description>IDE Controller</rasd:Description>
<rasd:ElementName>VirtualIDEController 0</rasd:ElementName>
<rasd:InstanceID>4</rasd:InstanceID>
<rasd:ResourceType>5</rasd:ResourceType>
</Item>
<Item ovf:required="false">
<rasd:AutomaticAllocation>false</rasd:AutomaticAllocation>
<rasd:ElementName>VirtualVideoCard</rasd:ElementName>
<rasd:InstanceID>5</rasd:InstanceID>
<rasd:ResourceType>24</rasd:ResourceType>
<vmw:Config ovf:required="false" vmw:key="enable3DSupport" vmw:value="false" />
<vmw:Config ovf:required="false" vmw:key="enableMPTSupport" vmw:value="false" />
<vmw:Config ovf:required="false" vmw:key="use3dRenderer" vmw:value="automatic" />
<vmw:Config ovf:required="false" vmw:key="useAutoDetect" vmw:value="false" />
<vmw:Config ovf:required="false" vmw:key="videoRamSizeInKB" vmw:value="4096" />
</Item>
<Item ovf:required="false">
<rasd:AutomaticAllocation>false</rasd:AutomaticAllocation>
<rasd:ElementName>VirtualVMCIDevice</rasd:ElementName>
<rasd:InstanceID>6</rasd:InstanceID>
<rasd:ResourceSubType>vmware.vmci</rasd:ResourceSubType>
<rasd:ResourceType>1</rasd:ResourceType>
<vmw:Config ovf:required="false" vmw:key="allowUnrestrictedCommunication" vmw:value="false" />
<vmw:Config ovf:required="false" vmw:key="slotInfo.pciSlotNumber" vmw:value="33" />
</Item>
<Item>
<rasd:AddressOnParent>1</rasd:AddressOnParent>
<rasd:AutomaticAllocation>true</rasd:AutomaticAllocation>
<rasd:ElementName>CD-ROM 1</rasd:ElementName>
<rasd:HostResource>ovf:/file/file1</rasd:HostResource>
<rasd:InstanceID>7</rasd:InstanceID>
<rasd:Parent>4</rasd:Parent>
<rasd:ResourceSubType>vmware.cdrom.iso</rasd:ResourceSubType>
<rasd:ResourceType>15</rasd:ResourceType>
</Item>
<Item>
<rasd:AddressOnParent>0</rasd:AddressOnParent>
<rasd:ElementName>Hard Disk 1</rasd:ElementName>
<rasd:HostResource>ovf:/disk/vmdisk1</rasd:HostResource>
<rasd:InstanceID>8</rasd:InstanceID>
<rasd:Parent>4</rasd:Parent>
<rasd:ResourceType>17</rasd:ResourceType>
<vmw:Config ovf:required="false" vmw:key="backing.writeThrough" vmw:value="false" />
</Item>
<Item>
<rasd:AddressOnParent>7</rasd:AddressOnParent>
<rasd:AutomaticAllocation>true</rasd:AutomaticAllocation>
<rasd:Connection>dvportgroup-81</rasd:Connection>
<rasd:Description>E1000 ethernet adapter on "dvportgroup-81"</rasd:Description>
<rasd:ElementName>Ethernet 1</rasd:ElementName>
<rasd:InstanceID>9</rasd:InstanceID>
<rasd:ResourceSubType>E1000</rasd:ResourceSubType>
<rasd:ResourceType>10</rasd:ResourceType>
<vmw:Config ovf:required="false" vmw:key="slotInfo.pciSlotNumber" vmw:value="32" />
<vmw:Config ovf:required="false" vmw:key="wakeOnLanEnabled" vmw:value="true" />
</Item>
<vmw:Config ovf:required="false" vmw:key="cpuHotAddEnabled" vmw:value="false" />
<vmw:Config ovf:required="false" vmw:key="cpuHotRemoveEnabled" vmw:value="false" />
<vmw:Config ovf:required="false" vmw:key="firmware" vmw:value="bios" />
<vmw:Config ovf:required="false" vmw:key="virtualICH7MPresent" vmw:value="false" />
<vmw:Config ovf:required="false" vmw:key="virtualSMCPresent" vmw:value="false" />
<vmw:Config ovf:required="false" vmw:key="memoryHotAddEnabled" vmw:value="false" />
<vmw:Config ovf:required="false" vmw:key="nestedHVEnabled" vmw:value="false" />
<vmw:Config ovf:required="false" vmw:key="powerOpInfo.powerOffType" vmw:value="soft" />
<vmw:Config ovf:required="false" vmw:key="powerOpInfo.resetType" vmw:value="soft" />
<vmw:Config ovf:required="false" vmw:key="powerOpInfo.standbyAction" vmw:value="checkpoint" />
<vmw:Config ovf:required="false" vmw:key="powerOpInfo.suspendType" vmw:value="hard" />
<vmw:Config ovf:required="false" vmw:key="tools.afterPowerOn" vmw:value="true" />
<vmw:Config ovf:required="false" vmw:key="tools.afterResume" vmw:value="true" />
<vmw:Config ovf:required="false" vmw:key="tools.beforeGuestShutdown" vmw:value="true" />
<vmw:Config ovf:required="false" vmw:key="tools.beforeGuestStandby" vmw:value="true" />
<vmw:Config ovf:required="false" vmw:key="tools.syncTimeWithHost" vmw:value="false" />
<vmw:Config ovf:required="false" vmw:key="tools.toolsUpgradePolicy" vmw:value="manual" />
</VirtualHardwareSection>
<AnnotationSection ovf:required="false">
<Info>A human-readable annotation</Info>
<Annotation>foo</Annotation>
</AnnotationSection>
</VirtualSystem>
</Envelope>

177
oslo_vmware/tests/test_image_transfer.py
View File

@ -405,59 +405,148 @@ class ImageTransferUtilityTest(base.TestCase):
fake_VmdkWriteHandle.get_imported_vm.assert_called_once_with() fake_VmdkWriteHandle.get_imported_vm.assert_called_once_with()
@mock.patch('tarfile.open')
@mock.patch('oslo_vmware.image_util.get_vmdk_name_from_ovf')
def test_get_vmdk_handle(self, get_vmdk_name_from_ovf, tar_open):
ovf_info = mock.Mock()
ovf_info.name = 'test.ovf'
vmdk_info = mock.Mock()
vmdk_info.name = 'test.vmdk'
tar = mock.Mock()
tar.__iter__ = mock.Mock(return_value=iter([ovf_info, vmdk_info]))
tar.__enter__ = mock.Mock(return_value=tar)
tar.__exit__ = mock.Mock(return_value=None)
tar_open.return_value = tar
ovf_handle = mock.Mock()
get_vmdk_name_from_ovf.return_value = 'test.vmdk'
vmdk_handle = mock.Mock()
tar.extractfile.side_effect = [ovf_handle, vmdk_handle]
ova_handle = mock.sentinel.ova_handle
ret = image_transfer._get_vmdk_handle(ova_handle)
self.assertEqual(vmdk_handle, ret)
tar_open.assert_called_once_with(mode="r|", fileobj=ova_handle)
self.assertEqual([mock.call(ovf_info), mock.call(vmdk_info)],
tar.extractfile.call_args_list)
get_vmdk_name_from_ovf.assert_called_once_with(ovf_handle)
@mock.patch('tarfile.open')
def test_get_vmdk_handle_with_invalid_ova(self, tar_open):
tar = mock.Mock()
tar.__iter__ = mock.Mock(return_value=iter([]))
tar.__enter__ = mock.Mock(return_value=tar)
tar.__exit__ = mock.Mock(return_value=None)
tar_open.return_value = tar
ova_handle = mock.sentinel.ova_handle
ret = image_transfer._get_vmdk_handle(ova_handle)
self.assertIsNone(ret)
tar_open.assert_called_once_with(mode="r|", fileobj=ova_handle)
self.assertFalse(tar.extractfile.called)
@mock.patch('oslo_vmware.rw_handles.ImageReadHandle') @mock.patch('oslo_vmware.rw_handles.ImageReadHandle')
@mock.patch.object(image_transfer, 'download_stream_optimized_data') @mock.patch.object(image_transfer, 'download_stream_optimized_data')
def test_download_stream_optimized_image( @mock.patch.object(image_transfer, '_get_vmdk_handle')
self, fake_download_stream_optimized_data, def _test_download_stream_optimized_image(
fake_rw_handles_ImageReadHandle): self,
get_vmdk_handle,
download_stream_optimized_data,
image_read_handle,
container=None,
invalid_ova=False):
context = mock.Mock()
session = mock.Mock()
image_id = mock.Mock()
timeout_secs = 10
image_size = 1000
host = '127.0.0.1'
port = 443
resource_pool = 'rp-1'
vm_folder = 'folder-1'
vm_import_spec = None
fake_iter = 'fake_iter'
image_service = mock.Mock() image_service = mock.Mock()
image_service.download = mock.Mock() if container:
image_service.download.return_value = fake_iter image_service.show.return_value = {'container_format': container}
read_iter = mock.sentinel.read_iter
image_service.download.return_value = read_iter
read_handle = mock.sentinel.read_handle
image_read_handle.return_value = read_handle
fake_ImageReadHandle = 'fake_ImageReadHandle' if container == 'ova':
fake_rw_handles_ImageReadHandle.return_value = fake_ImageReadHandle if invalid_ova:
get_vmdk_handle.return_value = None
else:
vmdk_handle = mock.sentinel.vmdk_handle
get_vmdk_handle.return_value = vmdk_handle
image_transfer.download_stream_optimized_image( imported_vm = mock.sentinel.imported_vm
context, download_stream_optimized_data.return_value = imported_vm
timeout_secs,
image_service,
image_id,
session=session,
host=host,
port=port,
resource_pool=resource_pool,
vm_folder=vm_folder,
vm_import_spec=vm_import_spec,
image_size=image_size)
image_service.download.assert_called_once_with(context, image_id) context = mock.sentinel.context
timeout_secs = mock.sentinel.timeout_secs
image_id = mock.sentinel.image_id
session = mock.sentinel.session
image_size = mock.sentinel.image_size
host = mock.sentinel.host
port = mock.sentinel.port
resource_pool = mock.sentinel.port
vm_folder = mock.sentinel.vm_folder
vm_import_spec = mock.sentinel.vm_import_spec
fake_rw_handles_ImageReadHandle.assert_called_once_with(fake_iter) if container == 'ova' and invalid_ova:
self.assertRaises(exceptions.ImageTransferException,
image_transfer.download_stream_optimized_image,
context,
timeout_secs,
image_service,
image_id,
session=session,
host=host,
port=port,
resource_pool=resource_pool,
vm_folder=vm_folder,
vm_import_spec=vm_import_spec,
image_size=image_size)
else:
ret = image_transfer.download_stream_optimized_image(
context,
timeout_secs,
image_service,
image_id,
session=session,
host=host,
port=port,
resource_pool=resource_pool,
vm_folder=vm_folder,
vm_import_spec=vm_import_spec,
image_size=image_size)
fake_download_stream_optimized_data.assert_called_once_with( self.assertEqual(imported_vm, ret)
context, image_service.show.assert_called_once_with(context, image_id)
timeout_secs, image_service.download.assert_called_once_with(context, image_id)
fake_ImageReadHandle, image_read_handle.assert_called_once_with(read_iter)
session=session, if container == 'ova':
host=host, get_vmdk_handle.assert_called_once_with(read_handle)
port=port, exp_read_handle = vmdk_handle
resource_pool=resource_pool, else:
vm_folder=vm_folder, exp_read_handle = read_handle
vm_import_spec=vm_import_spec, download_stream_optimized_data.assert_called_once_with(
image_size=image_size) context,
timeout_secs,
exp_read_handle,
session=session,
host=host,
port=port,
resource_pool=resource_pool,
vm_folder=vm_folder,
vm_import_spec=vm_import_spec,
image_size=image_size)
def test_download_stream_optimized_image(self):
self._test_download_stream_optimized_image()
def test_download_stream_optimized_image_ova(self):
self._test_download_stream_optimized_image(container='ova')
def test_download_stream_optimized_image_invalid_ova(self):
self._test_download_stream_optimized_image(container='ova',
invalid_ova=True)
@mock.patch.object(image_transfer, '_start_transfer') @mock.patch.object(image_transfer, '_start_transfer')
@mock.patch('oslo_vmware.rw_handles.VmdkReadHandle') @mock.patch('oslo_vmware.rw_handles.VmdkReadHandle')

32
oslo_vmware/tests/test_image_util.py Normal file
View File

@ -0,0 +1,32 @@
# Copyright (c) 2016 VMware, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Unit tests for image_util.
"""
import os
from oslo_vmware import image_util
from oslo_vmware.tests import base
class ImageUtilTest(base.TestCase):
def test_get_vmdk_name_from_ovf(self):
ovf_descriptor = os.path.join(os.path.dirname(__file__), 'test.ovf')
with open(ovf_descriptor) as f:
vmdk_name = image_util.get_vmdk_name_from_ovf(f)
self.assertEqual("test-disk1.vmdk", vmdk_name)

1
requirements.txt
View File

@ -14,6 +14,7 @@ oslo.utils>=3.5.0 # Apache-2.0
# for the routing notifier # for the routing notifier
PyYAML>=3.1.0 # MIT PyYAML>=3.1.0 # MIT
lxml>=2.3 # BSD
suds-jurko>=0.6 # LGPL suds-jurko>=0.6 # LGPL
eventlet!=0.18.3,>=0.18.2 # MIT eventlet!=0.18.3,>=0.18.2 # MIT
requests>=2.10.0 # Apache-2.0 requests>=2.10.0 # Apache-2.0