# Copyright (c) 2012 VMware, Inc. # Copyright (c) 2011 Citrix Systems, Inc. # Copyright 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """ Utility functions for Image transfer and manipulation. """ import os import tarfile from lxml import etree from oslo_config import cfg from oslo_log import log as logging from oslo_service import loopingcall from oslo_utils import encodeutils from oslo_utils import strutils from oslo_utils import units from oslo_vmware import rw_handles from nova import exception from nova.i18n import _ from nova.image import glance from nova.objects import fields from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import vm_util # NOTE(mdbooth): We use use_linked_clone below, but don't have to import it # because nova.virt.vmwareapi.driver is imported first. In fact, it is not # possible to import it here, as nova.virt.vmwareapi.driver calls # CONF.register_opts() after the import chain which imports this module. This # is not a problem as long as the import order doesn't change. CONF = cfg.CONF LOG = logging.getLogger(__name__) IMAGE_API = glance.API() QUEUE_BUFFER_SIZE = 10 NFC_LEASE_UPDATE_PERIOD = 60 # update NFC lease every 60sec. CHUNK_SIZE = 64 * units.Ki # default chunk size for image transfer class VMwareImage(object): def __init__(self, image_id, file_size=0, os_type=constants.DEFAULT_OS_TYPE, adapter_type=constants.DEFAULT_ADAPTER_TYPE, disk_type=constants.DEFAULT_DISK_TYPE, container_format=constants.CONTAINER_FORMAT_BARE, file_type=constants.DEFAULT_DISK_FORMAT, linked_clone=None, vsphere_location=None, vif_model=constants.DEFAULT_VIF_MODEL): """VMwareImage holds values for use in building VMs. image_id (str): uuid of the image file_size (int): size of file in bytes os_type (str): name of guest os (use vSphere names only) adapter_type (str): name of the adapter's type disk_type (str): type of disk in thin, thick, etc container_format (str): container format (bare or ova) file_type (str): vmdk or iso linked_clone (bool): use linked clone, or don't vsphere_location (str): image location in datastore or None vif_model (str): virtual machine network interface """ self.image_id = image_id self.file_size = file_size self.os_type = os_type self.adapter_type = adapter_type self.container_format = container_format self.disk_type = disk_type self.file_type = file_type self.vsphere_location = vsphere_location # NOTE(vui): This should be removed when we restore the # descriptor-based validation. if (self.file_type is not None and self.file_type not in constants.DISK_FORMATS_ALL): raise exception.InvalidDiskFormat(disk_format=self.file_type) if linked_clone is not None: self.linked_clone = linked_clone else: self.linked_clone = CONF.vmware.use_linked_clone self.vif_model = vif_model @property def file_size_in_kb(self): return self.file_size / units.Ki @property def is_sparse(self): return self.disk_type == constants.DISK_TYPE_SPARSE @property def is_iso(self): return self.file_type == constants.DISK_FORMAT_ISO @property def is_ova(self): return self.container_format == constants.CONTAINER_FORMAT_OVA @classmethod def from_image(cls, context, image_id, image_meta): """Returns VMwareImage, the subset of properties the driver uses. :param context - context :param image_id - image id of image :param image_meta - image metadata object we are working with :return: vmware image object :rtype: nova.virt.vmwareapi.images.VmwareImage """ properties = image_meta.properties # calculate linked_clone flag, allow image properties to override the # global property set in the configurations. image_linked_clone = properties.get('img_linked_clone', CONF.vmware.use_linked_clone) # catch any string values that need to be interpreted as boolean values linked_clone = strutils.bool_from_string(image_linked_clone) if image_meta.obj_attr_is_set('container_format'): container_format = image_meta.container_format else: container_format = None props = { 'image_id': image_id, 'linked_clone': linked_clone, 'container_format': container_format, 'vsphere_location': get_vsphere_location(context, image_id) } if image_meta.obj_attr_is_set('size'): props['file_size'] = image_meta.size if image_meta.obj_attr_is_set('disk_format'): props['file_type'] = image_meta.disk_format hw_disk_bus = properties.get('hw_disk_bus') if hw_disk_bus: mapping = { fields.SCSIModel.LSILOGIC: constants.DEFAULT_ADAPTER_TYPE, fields.SCSIModel.LSISAS1068: constants.ADAPTER_TYPE_LSILOGICSAS, fields.SCSIModel.BUSLOGIC: constants.ADAPTER_TYPE_BUSLOGIC, fields.SCSIModel.VMPVSCSI: constants.ADAPTER_TYPE_PARAVIRTUAL, } if hw_disk_bus == fields.DiskBus.IDE: props['adapter_type'] = constants.ADAPTER_TYPE_IDE elif hw_disk_bus == fields.DiskBus.SCSI: hw_scsi_model = properties.get('hw_scsi_model') props['adapter_type'] = mapping.get(hw_scsi_model) props_map = { 'os_distro': 'os_type', 'hw_disk_type': 'disk_type', 'hw_vif_model': 'vif_model' } for k, v in props_map.items(): if properties.obj_attr_is_set(k): props[v] = properties.get(k) return cls(**props) def get_vsphere_location(context, image_id): """Get image location in vsphere or None.""" # image_id can be None if the instance is booted using a volume. if image_id: metadata = IMAGE_API.get(context, image_id, include_locations=True) locations = metadata.get('locations') if locations: for loc in locations: loc_url = loc.get('url') if loc_url and loc_url.startswith('vsphere://'): return loc_url return None def image_transfer(read_handle, write_handle): # write_handle could be an NFC lease, so we need to periodically # update its progress update_cb = getattr(write_handle, 'update_progress', lambda: None) updater = loopingcall.FixedIntervalLoopingCall(update_cb) try: updater.start(interval=NFC_LEASE_UPDATE_PERIOD) while True: data = read_handle.read(CHUNK_SIZE) if not data: break write_handle.write(data) finally: updater.stop() read_handle.close() write_handle.close() def upload_iso_to_datastore(iso_path, instance, **kwargs): LOG.debug("Uploading iso %s to datastore", iso_path, instance=instance) with open(iso_path, 'r') as iso_file: write_file_handle = rw_handles.FileWriteHandle( kwargs.get("host"), kwargs.get("port"), kwargs.get("data_center_name"), kwargs.get("datastore_name"), kwargs.get("cookies"), kwargs.get("file_path"), os.fstat(iso_file.fileno()).st_size) LOG.debug("Uploading iso of size : %s ", os.fstat(iso_file.fileno()).st_size) block_size = 0x10000 data = iso_file.read(block_size) while len(data) > 0: write_file_handle.write(data) data = iso_file.read(block_size) write_file_handle.close() LOG.debug("Uploaded iso %s to datastore", iso_path, instance=instance) def fetch_image(context, instance, host, port, dc_name, ds_name, file_path, cookies=None): """Download image from the glance image server.""" image_ref = instance.image_ref LOG.debug("Downloading image file data %(image_ref)s to the " "data store %(data_store_name)s", {'image_ref': image_ref, 'data_store_name': ds_name}, instance=instance) metadata = IMAGE_API.get(context, image_ref) file_size = int(metadata['size']) read_iter = IMAGE_API.download(context, image_ref) read_file_handle = rw_handles.ImageReadHandle(read_iter) write_file_handle = rw_handles.FileWriteHandle( host, port, dc_name, ds_name, cookies, file_path, file_size) image_transfer(read_file_handle, write_file_handle) LOG.debug("Downloaded image file data %(image_ref)s to " "%(upload_name)s on the data store " "%(data_store_name)s", {'image_ref': image_ref, 'upload_name': 'n/a' if file_path is None else file_path, 'data_store_name': 'n/a' if ds_name is None else ds_name}, instance=instance) def _build_shadow_vm_config_spec(session, name, size_kb, disk_type, ds_name): """Return spec for creating a shadow VM for image disk. The VM is never meant to be powered on. When used in importing a disk it governs the directory name created for the VM and the disk type of the disk image to convert to. :param name: Name of the backing :param size_kb: Size in KB of the backing :param disk_type: VMDK type for the disk :param ds_name: Datastore name where the disk is to be provisioned :return: Spec for creation """ cf = session.vim.client.factory controller_device = cf.create('ns0:VirtualLsiLogicController') controller_device.key = -100 controller_device.busNumber = 0 controller_device.sharedBus = 'noSharing' controller_spec = cf.create('ns0:VirtualDeviceConfigSpec') controller_spec.operation = 'add' controller_spec.device = controller_device disk_device = cf.create('ns0:VirtualDisk') # for very small disks allocate at least 1KB disk_device.capacityInKB = max(1, int(size_kb)) disk_device.key = -101 disk_device.unitNumber = 0 disk_device.controllerKey = -100 disk_device_bkng = cf.create('ns0:VirtualDiskFlatVer2BackingInfo') if disk_type == constants.DISK_TYPE_EAGER_ZEROED_THICK: disk_device_bkng.eagerlyScrub = True elif disk_type == constants.DISK_TYPE_THIN: disk_device_bkng.thinProvisioned = True disk_device_bkng.fileName = '[%s]' % ds_name disk_device_bkng.diskMode = 'persistent' disk_device.backing = disk_device_bkng disk_spec = cf.create('ns0:VirtualDeviceConfigSpec') disk_spec.operation = 'add' disk_spec.fileOperation = 'create' disk_spec.device = disk_device vm_file_info = cf.create('ns0:VirtualMachineFileInfo') vm_file_info.vmPathName = '[%s]' % ds_name create_spec = cf.create('ns0:VirtualMachineConfigSpec') create_spec.name = name create_spec.guestId = constants.DEFAULT_OS_TYPE create_spec.numCPUs = 1 create_spec.memoryMB = 128 create_spec.deviceChange = [controller_spec, disk_spec] create_spec.files = vm_file_info return create_spec def _build_import_spec_for_import_vapp(session, vm_name, datastore_name): vm_create_spec = _build_shadow_vm_config_spec( session, vm_name, 0, constants.DISK_TYPE_THIN, datastore_name) client_factory = session.vim.client.factory vm_import_spec = client_factory.create('ns0:VirtualMachineImportSpec') vm_import_spec.configSpec = vm_create_spec return vm_import_spec def fetch_image_stream_optimized(context, instance, session, vm_name, ds_name, vm_folder_ref, res_pool_ref): """Fetch image from Glance to ESX datastore.""" image_ref = instance.image_ref LOG.debug("Downloading image file data %(image_ref)s to the ESX " "as VM named '%(vm_name)s'", {'image_ref': image_ref, 'vm_name': vm_name}, instance=instance) metadata = IMAGE_API.get(context, image_ref) file_size = int(metadata['size']) vm_import_spec = _build_import_spec_for_import_vapp( session, vm_name, ds_name) read_iter = IMAGE_API.download(context, image_ref) read_handle = rw_handles.ImageReadHandle(read_iter) write_handle = rw_handles.VmdkWriteHandle(session, session._host, session._port, res_pool_ref, vm_folder_ref, vm_import_spec, file_size) image_transfer(read_handle, write_handle) imported_vm_ref = write_handle.get_imported_vm() LOG.info("Downloaded image file data %(image_ref)s", {'image_ref': instance.image_ref}, instance=instance) vmdk = vm_util.get_vmdk_info(session, imported_vm_ref, vm_name) session._call_method(session.vim, "UnregisterVM", imported_vm_ref) LOG.info("The imported VM was unregistered", instance=instance) return vmdk.capacity_in_bytes def get_vmdk_name_from_ovf(xmlstr): """Parse the OVA descriptor to extract the vmdk name.""" ovf = etree.fromstring(encodeutils.safe_encode(xmlstr)) nsovf = "{%s}" % ovf.nsmap["ovf"] disk = ovf.find("./%sDiskSection/%sDisk" % (nsovf, nsovf)) file_id = disk.get("%sfileRef" % nsovf) file = ovf.find('./%sReferences/%sFile[@%sid="%s"]' % (nsovf, nsovf, nsovf, file_id)) vmdk_name = file.get("%shref" % nsovf) return vmdk_name def fetch_image_ova(context, instance, session, vm_name, ds_name, vm_folder_ref, res_pool_ref): """Download the OVA image from the glance image server to the Nova compute node. """ image_ref = instance.image_ref LOG.debug("Downloading OVA image file %(image_ref)s to the ESX " "as VM named '%(vm_name)s'", {'image_ref': image_ref, 'vm_name': vm_name}, instance=instance) metadata = IMAGE_API.get(context, image_ref) file_size = int(metadata['size']) vm_import_spec = _build_import_spec_for_import_vapp( session, vm_name, ds_name) read_iter = IMAGE_API.download(context, image_ref) read_handle = rw_handles.ImageReadHandle(read_iter) with tarfile.open(mode="r|", fileobj=read_handle) as tar: vmdk_name = None for tar_info in tar: if tar_info and tar_info.name.endswith(".ovf"): extracted = tar.extractfile(tar_info) xmlstr = extracted.read() vmdk_name = get_vmdk_name_from_ovf(xmlstr) elif vmdk_name and tar_info.name.startswith(vmdk_name): # Actual file name is .XXXXXXX extracted = tar.extractfile(tar_info) write_handle = rw_handles.VmdkWriteHandle( session, session._host, session._port, res_pool_ref, vm_folder_ref, vm_import_spec, file_size) image_transfer(extracted, write_handle) LOG.info("Downloaded OVA image file %(image_ref)s", {'image_ref': instance.image_ref}, instance=instance) imported_vm_ref = write_handle.get_imported_vm() vmdk = vm_util.get_vmdk_info(session, imported_vm_ref, vm_name) session._call_method(session.vim, "UnregisterVM", imported_vm_ref) LOG.info("The imported VM was unregistered", instance=instance) return vmdk.capacity_in_bytes raise exception.ImageUnacceptable( reason=_("Extracting vmdk from OVA failed."), image_id=image_ref) def upload_image_stream_optimized(context, image_id, instance, session, vm, vmdk_size): """Upload the snapshotted vm disk file to Glance image server.""" LOG.debug("Uploading image %s", image_id, instance=instance) metadata = IMAGE_API.get(context, image_id) read_handle = rw_handles.VmdkReadHandle(session, session._host, session._port, vm, None, vmdk_size) # Set the image properties. It is important to set the 'size' to 0. # Otherwise, the image service client will use the VM's disk capacity # which will not be the image size after upload, since it is converted # to a stream-optimized sparse disk. image_metadata = {'disk_format': constants.DISK_FORMAT_VMDK, 'name': metadata['name'], 'status': 'active', 'container_format': constants.CONTAINER_FORMAT_BARE, 'size': 0, 'properties': {'vmware_image_version': 1, 'vmware_disktype': 'streamOptimized', 'owner_id': instance.project_id}} updater = loopingcall.FixedIntervalLoopingCall(read_handle.update_progress) try: updater.start(interval=NFC_LEASE_UPDATE_PERIOD) IMAGE_API.update(context, image_id, image_metadata, data=read_handle) finally: updater.stop() read_handle.close() LOG.debug("Uploaded image %s to the Glance image server", image_id, instance=instance)