From bf882fbc4aa68cf0191bce70e622d81969617c02 Mon Sep 17 00:00:00 2001 From: Gary Kotton Date: Tue, 24 Dec 2013 06:19:26 -0800 Subject: [PATCH] VMware: create datastore utility functions The patch moves datastore interfaces from vmops and vm_util to a separate file - ds_util. This is part of the blueprint vmware-image-cache-management. In the past the datastore methods would poll the task to determine that it had completed. This has beem addressed by making the driver _poll_task return the task information. The patch set also solves the following bugs by checking if the FileAlreadyExistsException exception is raised. This could be due to another process or thread creating the file. Closes-bug: 1230047 Closes-bug: 1254128 Change-Id: I788e33dbcb3dedc41831b976137607274b1c02ca --- nova/tests/virt/vmwareapi/test_driver_api.py | 34 ++- nova/tests/virt/vmwareapi/test_ds_util.py | 182 +++++++++++++++ nova/tests/virt/vmwareapi/test_vmops.py | 56 ++--- nova/virt/vmwareapi/driver.py | 2 +- nova/virt/vmwareapi/ds_util.py | 126 +++++++++++ nova/virt/vmwareapi/fake.py | 3 +- nova/virt/vmwareapi/vm_util.py | 18 -- nova/virt/vmwareapi/vmops.py | 225 +++++++------------ 8 files changed, 446 insertions(+), 200 deletions(-) create mode 100644 nova/tests/virt/vmwareapi/test_ds_util.py create mode 100644 nova/virt/vmwareapi/ds_util.py diff --git a/nova/tests/virt/vmwareapi/test_driver_api.py b/nova/tests/virt/vmwareapi/test_driver_api.py index ce13193e2085..ebd169810036 100644 --- a/nova/tests/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/virt/vmwareapi/test_driver_api.py @@ -49,6 +49,7 @@ from nova import utils as nova_utils from nova.virt import driver as v_driver from nova.virt import fake from nova.virt.vmwareapi import driver +from nova.virt.vmwareapi import ds_util from nova.virt.vmwareapi import error_util from nova.virt.vmwareapi import fake as vmwareapi_fake from nova.virt.vmwareapi import vim @@ -269,6 +270,7 @@ class VMwareAPIVMTestCase(test.NoDBTestCase): host_password='test_pass', cluster_name='test_cluster', datastore_regex='.*', + api_retry_count=1, use_linked_clone=False, group='vmware') self.flags(vnc_enabled=False, image_cache_subdirectory_name='vmware_base') @@ -318,8 +320,13 @@ class VMwareAPIVMTestCase(test.NoDBTestCase): raise exception.NovaException('Here is my fake exception') return self.login_session + def _fake_check_session(_self): + return True + self.stubs.Set(vmwareapi_fake.FakeVim, '_login', _fake_login) self.stubs.Set(time, 'sleep', lambda x: None) + self.stubs.Set(vmwareapi_fake.FakeVim, '_check_session', + _fake_check_session) self.conn = driver.VMwareAPISession() self.assertEqual(self.attempts, 2) @@ -552,6 +559,23 @@ class VMwareAPIVMTestCase(test.NoDBTestCase): 'node': self.instance_node}) self._check_vm_info(info, power_state.RUNNING) + def test_spawn_disk_extend_exists(self): + root = ('[%s] vmware_base/fake_image_uuid/fake_image_uuid.80.vmdk' % + self.ds) + self.root = root + + def _fake_extend(instance, requested_size, name, dc_ref): + vmwareapi_fake._add_file(self.root) + + self.stubs.Set(self.conn._vmops, '_extend_virtual_disk', + _fake_extend) + + self._create_vm() + info = self.conn.get_info({'uuid': self.uuid, + 'node': self.instance_node}) + self._check_vm_info(info, power_state.RUNNING) + self.assertTrue(vmwareapi_fake.get_file(root)) + def test_spawn_disk_extend_sparse(self): self.mox.StubOutWithMock(vmware_images, 'get_vmdk_size_and_properties') result = [1024, {"vmware_ostype": "otherGuest", @@ -1512,6 +1536,7 @@ class VMwareAPIVCDriverTestCase(VMwareAPIVMTestCase): cluster_name = 'test_cluster' cluster_name2 = 'test_cluster2' self.flags(cluster_name=[cluster_name, cluster_name2], + api_retry_count=1, task_poll_interval=10, datastore_regex='.*', group='vmware') self.flags(vnc_enabled=False, image_cache_subdirectory_name='vmware_base') @@ -1631,15 +1656,12 @@ class VMwareAPIVCDriverTestCase(VMwareAPIVMTestCase): 'generate_uuid') uuidutils.generate_uuid().AndReturn(uuid_str) - self.mox.StubOutWithMock(vmops.VMwareVMOps, - '_delete_datastore_file') + self.mox.StubOutWithMock(ds_util, 'file_delete') # Check calls for delete vmdk and -flat.vmdk pair - self.conn._vmops._delete_datastore_file( - mox.IgnoreArg(), + ds_util.file_delete(mox.IgnoreArg(), "[%s] vmware_temp/%s-flat.vmdk" % (self.ds, uuid_str), mox.IgnoreArg()).AndReturn(None) - self.conn._vmops._delete_datastore_file( - mox.IgnoreArg(), + ds_util.file_delete(mox.IgnoreArg(), "[%s] vmware_temp/%s.vmdk" % (self.ds, uuid_str), mox.IgnoreArg()).AndReturn(None) diff --git a/nova/tests/virt/vmwareapi/test_ds_util.py b/nova/tests/virt/vmwareapi/test_ds_util.py new file mode 100644 index 000000000000..d80eeb0888f3 --- /dev/null +++ b/nova/tests/virt/vmwareapi/test_ds_util.py @@ -0,0 +1,182 @@ +# Copyright (c) 2014 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import contextlib +import mock + +from nova import test +from nova.virt.vmwareapi import ds_util +from nova.virt.vmwareapi import error_util +from nova.virt.vmwareapi import fake + + +class fake_session(object): + def __init__(self, ret=None): + self.ret = ret + + def _get_vim(self): + return fake.FakeVim() + + def _call_method(self, module, method, *args, **kwargs): + return self.ret + + def _wait_for_task(self, task_ref): + task_info = self._call_method('module', "get_dynamic_property", + task_ref, "Task", "info") + task_name = task_info.name + if task_info.state == 'success': + return task_info + else: + error_info = 'fake error' + error = task_info.error + name = error.fault.__class__.__name__ + raise error_util.get_fault_class(name)(error_info) + + +class DsUtilTestCase(test.NoDBTestCase): + def setUp(self): + super(DsUtilTestCase, self).setUp() + self.session = fake_session() + self.flags(api_retry_count=1, group='vmware') + fake.reset() + + def tearDown(self): + super(DsUtilTestCase, self).tearDown() + fake.reset() + + def test_build_datastore_path(self): + path = ds_util.build_datastore_path('ds', 'folder') + self.assertEqual('[ds] folder', path) + path = ds_util.build_datastore_path('ds', 'folder/file') + self.assertEqual('[ds] folder/file', path) + + def test_split_datastore_path(self): + ds, path = ds_util.split_datastore_path('[ds]') + self.assertEqual('ds', ds) + self.assertEqual('', path) + ds, path = ds_util.split_datastore_path('[ds] folder') + self.assertEqual('ds', ds) + self.assertEqual('folder', path) + ds, path = ds_util.split_datastore_path('[ds] folder/file') + self.assertEqual('ds', ds) + self.assertEqual('folder/file', path) + self.assertRaises(IndexError, + ds_util.split_datastore_path, + 'split bad path') + + def test_file_delete(self): + def fake_call_method(module, method, *args, **kwargs): + self.assertEqual('DeleteDatastoreFile_Task', method) + name = kwargs.get('name') + self.assertEqual('fake-datastore-path', name) + datacenter = kwargs.get('datacenter') + self.assertEqual('fake-dc-ref', datacenter) + return 'fake_delete_task' + + with contextlib.nested( + mock.patch.object(self.session, '_wait_for_task'), + mock.patch.object(self.session, '_call_method', + fake_call_method) + ) as (_wait_for_task, _call_method): + ds_util.file_delete(self.session, + 'fake-datastore-path', 'fake-dc-ref') + _wait_for_task.assert_has_calls([ + mock.call('fake_delete_task')]) + + def test_file_move(self): + def fake_call_method(module, method, *args, **kwargs): + self.assertEqual('MoveDatastoreFile_Task', method) + sourceName = kwargs.get('sourceName') + self.assertEqual('[ds] tmp/src', sourceName) + destinationName = kwargs.get('destinationName') + self.assertEqual('[ds] base/dst', destinationName) + sourceDatacenter = kwargs.get('sourceDatacenter') + self.assertEqual('fake-dc-ref', sourceDatacenter) + destinationDatacenter = kwargs.get('destinationDatacenter') + self.assertEqual('fake-dc-ref', destinationDatacenter) + return 'fake_move_task' + + with contextlib.nested( + mock.patch.object(self.session, '_wait_for_task'), + mock.patch.object(self.session, '_call_method', + fake_call_method) + ) as (_wait_for_task, _call_method): + ds_util.file_move(self.session, + 'fake-dc-ref', '[ds] tmp/src', '[ds] base/dst') + _wait_for_task.assert_has_calls([ + mock.call('fake_move_task')]) + + def test_mkdir(self): + def fake_call_method(module, method, *args, **kwargs): + self.assertEqual('MakeDirectory', method) + name = kwargs.get('name') + self.assertEqual('fake-path', name) + datacenter = kwargs.get('datacenter') + self.assertEqual('fake-dc-ref', datacenter) + createParentDirectories = kwargs.get('createParentDirectories') + self.assertTrue(createParentDirectories) + + with mock.patch.object(self.session, '_call_method', + fake_call_method): + ds_util.mkdir(self.session, 'fake-path', 'fake-dc-ref') + + def test_file_exists(self): + def fake_call_method(module, method, *args, **kwargs): + if method == 'SearchDatastore_Task': + ds_browser = args[0] + self.assertEqual('fake-browser', ds_browser) + datastorePath = kwargs.get('datastorePath') + self.assertEqual('fake-path', datastorePath) + return 'fake_exists_task' + elif method == 'get_dynamic_property': + info = fake.DataObject() + info.name = 'search_task' + info.state = 'success' + result = fake.DataObject() + result.path = 'fake-path' + matched = fake.DataObject() + matched.path = 'fake-file' + result.file = [matched] + info.result = result + return info + # Should never get here + self.fail() + + with mock.patch.object(self.session, '_call_method', + fake_call_method): + file_exists = ds_util.file_exists(self.session, + 'fake-browser', 'fake-path', 'fake-file') + self.assertTrue(file_exists) + + def test_file_exists_fails(self): + def fake_call_method(module, method, *args, **kwargs): + if method == 'SearchDatastore_Task': + return 'fake_exists_task' + elif method == 'get_dynamic_property': + info = fake.DataObject() + info.name = 'search_task' + info.state = 'error' + error = fake.DataObject() + error.localizedMessage = "Error message" + error.fault = fake.FileNotFound() + info.error = error + return info + # Should never get here + self.fail() + + with mock.patch.object(self.session, '_call_method', + fake_call_method): + file_exists = ds_util.file_exists(self.session, + 'fake-browser', 'fake-path', 'fake-file') + self.assertFalse(file_exists) diff --git a/nova/tests/virt/vmwareapi/test_vmops.py b/nova/tests/virt/vmwareapi/test_vmops.py index b71506042511..d7c0b00790a8 100644 --- a/nova/tests/virt/vmwareapi/test_vmops.py +++ b/nova/tests/virt/vmwareapi/test_vmops.py @@ -17,8 +17,8 @@ import mock from nova.network import model as network_model from nova import test from nova import utils +from nova.virt.vmwareapi import ds_util from nova.virt.vmwareapi import error_util -from nova.virt.vmwareapi import vm_util from nova.virt.vmwareapi import vmops @@ -105,41 +105,45 @@ class VMwareVMOpsTestCase(test.NoDBTestCase): self.assertTrue(value, "image level metadata failed to override global") - def _setup_create_cache_mocks(self): + def _setup_create_folder_mocks(self): ops = vmops.VMwareVMOps(mock.Mock(), mock.Mock(), mock.Mock()) - base_name = ops._base_folder + base_name = 'folder' ds_name = "datastore" ds_ref = mock.Mock() - path = vm_util.build_datastore_path(ds_name, base_name) - ops._mkdir = mock.Mock() - return ds_name, ds_ref, ops, path + ds_ref.value = 1 + dc_ref = mock.Mock() + ops._datastore_dc_mapping[ds_ref.value] = vmops.DcInfo( + ref=dc_ref, + name='fake-name', + vmFolder='fake-folder') + path = ds_util.build_datastore_path(ds_name, base_name) + ds_util.mkdir = mock.Mock() + return ds_name, ds_ref, ops, path, dc_ref - def test_create_cache_folder(self): - ds_name, ds_ref, ops, path = self._setup_create_cache_mocks() - ops.create_cache_folder(ds_name, ds_ref) - ops._mkdir.assert_called_with(path, ds_ref) + def test_create_folder_if_missing(self): + ds_name, ds_ref, ops, path, dc = self._setup_create_folder_mocks() + ops._create_folder_if_missing(ds_name, ds_ref, 'folder') + ds_util.mkdir.assert_called_with(ops._session, path, dc) - def test_create_cache_folder_with_exception(self): - ds_name, ds_ref, ops, path = self._setup_create_cache_mocks() - ops._mkdir.side_effect = error_util.FileAlreadyExistsException() - ops.create_cache_folder(ds_name, ds_ref) - # assert that the - ops._mkdir.assert_called_with(path, ds_ref) + def test_create_folder_if_missing_exception(self): + ds_name, ds_ref, ops, path, dc = self._setup_create_folder_mocks() + ds_util.mkdir.side_effect = error_util.FileAlreadyExistsException() + ops._create_folder_if_missing(ds_name, ds_ref, 'folder') + ds_util.mkdir.assert_called_with(ops._session, path, dc) - def test_check_if_folder_file_exists_with_existing(self): + @mock.patch.object(ds_util, 'file_exists', return_value=True) + def test_check_if_folder_file_exists_with_existing(self, + mock_exists): ops = vmops.VMwareVMOps(mock.Mock(), mock.Mock(), mock.Mock()) - ops.create_cache_folder = mock.Mock() - ops._file_exists = mock.Mock() - ops._file_exists.return_value = True + ops._create_folder_if_missing = mock.Mock() ops._check_if_folder_file_exists(mock.Mock(), "datastore", "folder", "some_file") - ops.create_cache_folder.assert_called_once() + ops._create_folder_if_missing.assert_called_once() - def test_check_if_folder_file_exists_no_existing(self): + @mock.patch.object(ds_util, 'file_exists', return_value=False) + def test_check_if_folder_file_exists_no_existing(self, mock_exists): ops = vmops.VMwareVMOps(mock.Mock(), mock.Mock(), mock.Mock()) - ops.create_cache_folder = mock.Mock() - ops._file_exists = mock.Mock() - ops._file_exists.return_value = True + ops._create_folder_if_missing = mock.Mock() ops._check_if_folder_file_exists(mock.Mock(), "datastore", "folder", "some_file") - ops.create_cache_folder.assert_called_once() + ops._create_folder_if_missing.assert_called_once() diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index e2dd02d774cf..319a19d77e21 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -894,7 +894,7 @@ class VMwareAPISession(object): LOG.debug(_("Task [%(task_name)s] %(task_ref)s " "status: success"), {'task_name': task_name, 'task_ref': task_ref}) - done.send("success") + done.send(task_info) else: error_info = str(task_info.error.localizedMessage) LOG.warn(_("Task [%(task_name)s] %(task_ref)s " diff --git a/nova/virt/vmwareapi/ds_util.py b/nova/virt/vmwareapi/ds_util.py new file mode 100644 index 000000000000..69e2366f570f --- /dev/null +++ b/nova/virt/vmwareapi/ds_util.py @@ -0,0 +1,126 @@ +# Copyright (c) 2014 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Datastore utility functions +""" + +from nova.openstack.common.gettextutils import _ +from nova.openstack.common import log as logging +from nova.virt.vmwareapi import error_util +from nova.virt.vmwareapi import vm_util + +LOG = logging.getLogger(__name__) + + +def build_datastore_path(datastore_name, path): + """Build the datastore compliant path.""" + return "[%s] %s" % (datastore_name, path) + + +def split_datastore_path(datastore_path): + """Return the datastore and path from a datastore_path. + + Split the VMware style datastore path to get the Datastore + name and the entity path. + """ + spl = datastore_path.split('[', 1)[1].split(']', 1) + path = "" + if len(spl) == 1: + datastore_name = spl[0] + else: + datastore_name, path = spl + return datastore_name, path.strip() + + +def file_delete(session, datastore_path, dc_ref): + LOG.debug(_("Deleting the datastore file %s"), datastore_path) + vim = session._get_vim() + file_delete_task = session._call_method( + session._get_vim(), + "DeleteDatastoreFile_Task", + vim.get_service_content().fileManager, + name=datastore_path, + datacenter=dc_ref) + session._wait_for_task(file_delete_task) + LOG.debug(_("Deleted the datastore file")) + + +def file_move(session, dc_ref, src_file, dst_file): + """Moves the source file or folder to the destination. + + The list of possible faults that the server can return on error + include: + - CannotAccessFile: Thrown if the source file or folder cannot be + moved because of insufficient permissions. + - FileAlreadyExists: Thrown if a file with the given name already + exists at the destination. + - FileFault: Thrown if there is a generic file error + - FileLocked: Thrown if the source file or folder is currently + locked or in use. + - FileNotFound: Thrown if the file or folder specified by sourceName + is not found. + - InvalidDatastore: Thrown if the operation cannot be performed on + the source or destination datastores. + - NoDiskSpace: Thrown if there is not enough space available on the + destination datastore. + - RuntimeFault: Thrown if any type of runtime fault is thrown that + is not covered by the other faults; for example, + a communication error. + """ + LOG.debug(_("Moving file from %(src)s to %(dst)s."), + {'src': src_file, 'dst': dst_file}) + vim = session._get_vim() + move_task = session._call_method( + session._get_vim(), + "MoveDatastoreFile_Task", + vim.get_service_content().fileManager, + sourceName=src_file, + sourceDatacenter=dc_ref, + destinationName=dst_file, + destinationDatacenter=dc_ref) + session._wait_for_task(move_task) + LOG.debug(_("File moved")) + + +def file_exists(session, ds_browser, ds_path, file_name): + """Check if the file exists on the datastore.""" + client_factory = session._get_vim().client.factory + search_spec = vm_util.search_datastore_spec(client_factory, file_name) + search_task = session._call_method(session._get_vim(), + "SearchDatastore_Task", + ds_browser, + datastorePath=ds_path, + searchSpec=search_spec) + try: + task_info = session._wait_for_task(search_task) + except error_util.FileNotFoundException: + return False + + file_exists = (getattr(task_info.result, 'file', False) and + task_info.result.file[0].path == file_name) + return file_exists + + +def mkdir(session, ds_path, dc_ref): + """Creates a directory at the path specified. If it is just "NAME", + then a directory with this name is created at the topmost level of the + DataStore. + """ + LOG.debug(_("Creating directory with path %s"), ds_path) + session._call_method(session._get_vim(), "MakeDirectory", + session._get_vim().get_service_content().fileManager, + name=ds_path, datacenter=dc_ref, + createParentDirectories=True) + LOG.debug(_("Created directory with path %s"), ds_path) diff --git a/nova/virt/vmwareapi/fake.py b/nova/virt/vmwareapi/fake.py index c8e1f96cb6b0..d8f7b7d87d9f 100644 --- a/nova/virt/vmwareapi/fake.py +++ b/nova/virt/vmwareapi/fake.py @@ -1132,7 +1132,8 @@ class FakeVim(object): task_mdo = create_task(method, state="success", result=result) return task_mdo.obj - task_mdo = create_task(method, "error") + task_mdo = create_task(method, "error", + error_fault=FileNotFound()) return task_mdo.obj def _move_file(self, method, *args, **kwargs): diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index cc069da8a9f9..3703e1ebaa37 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -89,24 +89,6 @@ def vm_ref_cache_from_name(func): VNC_CONFIG_KEY = 'config.extraConfig["RemoteDisplay.vnc.port"]' -def build_datastore_path(datastore_name, path): - """Build the datastore compliant path.""" - return "[%s] %s" % (datastore_name, path) - - -def split_datastore_path(datastore_path): - """Split the VMware style datastore path to get the Datastore - name and the entity path. - """ - spl = datastore_path.split('[', 1)[1].split(']', 1) - path = "" - if len(spl) == 1: - datastore_url = spl[0] - else: - datastore_url, path = spl - return datastore_url, path.strip() - - def get_vm_create_spec(client_factory, instance, name, data_store_name, vif_infos, os_type="otherGuest"): """Builds the VM Create spec.""" diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 52120768d98c..5b313a44ec77 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -22,7 +22,6 @@ Class for VM tasks like spawn, snapshot, suspend, resume etc. import collections import copy import os -import time from oslo.config import cfg @@ -41,6 +40,7 @@ from nova.openstack.common import uuidutils from nova import utils from nova.virt import configdrive from nova.virt import driver +from nova.virt.vmwareapi import ds_util from nova.virt.vmwareapi import error_util from nova.virt.vmwareapi import vif as vmwarevif from nova.virt.vmwareapi import vim @@ -98,6 +98,7 @@ class VMwareVMOps(object): self._poll_rescue_last_ran = None self._is_neutron = utils.is_neutron() self._datastore_dc_mapping = {} + self._datastore_browser_mapping = {} def list_instances(self): """Lists the VM instances that are registered with the ESX host.""" @@ -155,30 +156,17 @@ class VMwareVMOps(object): LOG.debug(_("Extended root virtual disk")) def _delete_datastore_file(self, instance, datastore_path, dc_ref): - LOG.debug(_("Deleting the datastore file %s") % datastore_path, - instance=instance) - vim = self._session._get_vim() - file_delete_task = self._session._call_method( - self._session._get_vim(), - "DeleteDatastoreFile_Task", - vim.get_service_content().fileManager, - name=datastore_path, - datacenter=dc_ref) try: - self._session._wait_for_task(file_delete_task) + ds_util.file_delete(self._session, datastore_path, dc_ref) except (error_util.CannotDeleteFileException, error_util.FileFaultException, error_util.FileLockedException, error_util.FileNotFoundException) as e: - # There may be more than one process or thread that tries - # to delete the file. LOG.debug(_("Unable to delete %(ds)s. There may be more than " "one process or thread that tries to delete the file. " "Exception: %(ex)s"), {'ds': datastore_path, 'ex': e}) - LOG.debug(_("Deleted the datastore file"), instance=instance) - def spawn(self, context, instance, image_meta, injected_files, admin_password, network_info, block_device_info=None, instance_name=None, power_on=True): @@ -340,12 +328,10 @@ class VMwareVMOps(object): # storage adapter type. # Here we assume thick provisioning and lsiLogic for the adapter # type - LOG.debug(_("Creating temporary folder for %(folder)s on " - "datastore %(datastore)s."), - {'folder': folder, 'datastore': data_store_name}) - self._mkdir(vm_util.build_datastore_path(data_store_name, - folder), - data_store_ref) + folder_path = ds_util.build_datastore_path(data_store_name, + folder) + LOG.debug(_("Creating temporary folder %s"), folder_path) + ds_util.mkdir(self._session, folder_path, dc_info.ref) LOG.debug(_("Creating Virtual Disk of size " "%(vmdk_file_size_in_kb)s KB and adapter type " "%(adapter_type)s on the ESX host local store " @@ -373,30 +359,6 @@ class VMwareVMOps(object): "data_store_name": data_store_name}, instance=instance) - def _move_image_to_cache_folder(tmp_folder, base_folder): - LOG.debug(_("Moving temporary folder %(tmp)s to cache " - "folder %(cache)s."), - {'tmp': tmp_folder, 'cache': base_folder}) - vmdk_move_task = self._session._call_method( - self._session._get_vim(), - "MoveDatastoreFile_Task", - service_content.fileManager, - sourceName=vm_util.build_datastore_path(data_store_name, - tmp_folder), - sourceDatacenter=dc_info.ref, - destinationName=vm_util.build_datastore_path(data_store_name, - base_folder), - destinationDatacenter=dc_info.ref) - try: - self._session._wait_for_task(vmdk_move_task) - except error_util.FileAlreadyExistsException: - # File move has failed. This may be due to the fact that a - # process or thread has already completed the opertaion. - # In the event of a FileAlreadyExists we continue, all other - # exceptions will be raised. - LOG.debug(_("File %(folder)s already exists on %(ds)s."), - {'folder': base_folder, 'ds': data_store_name}) - def _fetch_image_on_datastore(): """Fetch image from Glance to datastore.""" LOG.debug(_("Downloading image file data %(image_ref)s to the " @@ -475,7 +437,7 @@ class VMwareVMOps(object): # The vmdk meta-data file uploaded_vmdk_name = "%s/%s.vmdk" % (upload_folder, upload_name) - uploaded_vmdk_path = vm_util.build_datastore_path(data_store_name, + uploaded_vmdk_path = ds_util.build_datastore_path(data_store_name, uploaded_vmdk_name) session_vim = self._session._get_vim() @@ -498,15 +460,15 @@ class VMwareVMOps(object): sparse_uploaded_vmdk_name = "%s/%s-sparse.vmdk" % ( upload_folder, upload_name) - flat_uploaded_vmdk_path = vm_util.build_datastore_path( + flat_uploaded_vmdk_path = ds_util.build_datastore_path( data_store_name, flat_uploaded_vmdk_name) - sparse_uploaded_vmdk_path = vm_util.build_datastore_path( + sparse_uploaded_vmdk_path = ds_util.build_datastore_path( data_store_name, sparse_uploaded_vmdk_name) vmdk_name = "%s/%s.vmdk" % (upload_folder, upload_name) - vmdk_path = vm_util.build_datastore_path(data_store_name, + vmdk_path = ds_util.build_datastore_path(data_store_name, vmdk_name) if disk_type != "sparse": # Create a flat virtual disk and retain the metadata file. @@ -524,11 +486,24 @@ class VMwareVMOps(object): self._delete_datastore_file(instance, sparse_uploaded_vmdk_path, dc_info.ref) - base_folder = '%s/%s' % (self._base_folder, upload_name) - _move_image_to_cache_folder(upload_folder, base_folder) + dest_folder = ds_util.build_datastore_path(data_store_name, + base_folder) + src_folder = ds_util.build_datastore_path(data_store_name, + upload_folder) + try: + ds_util.file_move(self._session, dc_info.ref, + src_folder, dest_folder) + except error_util.FileAlreadyExistsException: + # File move has failed. This may be due to the fact that a + # process or thread has already completed the opertaion. + # In the event of a FileAlreadyExists we continue, + # all other exceptions will be raised. + LOG.debug(_("File %s already exists"), dest_folder) + + # Delete the temp upload folder self._delete_datastore_file(instance, - vm_util.build_datastore_path(data_store_name, + ds_util.build_datastore_path(data_store_name, tmp_upload_folder), dc_info.ref) else: @@ -545,7 +520,7 @@ class VMwareVMOps(object): dest_name = instance_name dest_vmdk_name = "%s/%s.vmdk" % (dest_folder, dest_name) - dest_vmdk_path = vm_util.build_datastore_path( + dest_vmdk_path = ds_util.build_datastore_path( data_store_name, dest_vmdk_name) _copy_virtual_disk(uploaded_vmdk_path, dest_vmdk_path) @@ -557,7 +532,7 @@ class VMwareVMOps(object): upload_folder = '%s/%s' % (self._base_folder, upload_name) root_vmdk_name = "%s/%s.%s.vmdk" % (upload_folder, upload_name, root_gb) - root_vmdk_path = vm_util.build_datastore_path(data_store_name, + root_vmdk_path = ds_util.build_datastore_path(data_store_name, root_vmdk_name) if not self._check_if_folder_file_exists( data_store_ref, data_store_name, @@ -598,7 +573,7 @@ class VMwareVMOps(object): dc_info.name, instance['uuid'], cookies) - uploaded_iso_path = vm_util.build_datastore_path( + uploaded_iso_path = ds_util.build_datastore_path( data_store_name, uploaded_iso_path) self._attach_cdrom_to_vm( @@ -773,7 +748,7 @@ class VMwareVMOps(object): (vmdk_file_path_before_snapshot, adapter_type, disk_type) = vm_util.get_vmdk_path_and_adapter_type( hw_devices, uuid=instance['uuid']) - datastore_name = vm_util.split_datastore_path( + datastore_name = ds_util.split_datastore_path( vmdk_file_path_before_snapshot)[0] os_type = self._session._call_method(vim_util, "get_dynamic_property", vm_ref, @@ -811,16 +786,7 @@ class VMwareVMOps(object): if ds_ref_ret is None: raise exception.DatastoreNotFound() ds_ref = ds_ref_ret.ManagedObjectReference[0] - ds_browser = self._session._call_method( - vim_util, "get_dynamic_property", ds_ref, "Datastore", - "browser") - # Check if the vmware-tmp folder exists or not. If not, create one - tmp_folder_path = vm_util.build_datastore_path(datastore_name, - self._tmp_folder) - if not self._path_exists(ds_browser, tmp_folder_path): - self._mkdir(vm_util.build_datastore_path(datastore_name, - self._tmp_folder), - ds_ref) + self.check_temp_folder(datastore_name, ds_ref) return ds_ref ds_ref = _check_if_tmp_folder_exists() @@ -829,9 +795,9 @@ class VMwareVMOps(object): # will be copied to. A random name is chosen so that we don't have # name clashes. random_name = uuidutils.generate_uuid() - dest_vmdk_file_path = vm_util.build_datastore_path(datastore_name, + dest_vmdk_file_path = ds_util.build_datastore_path(datastore_name, "%s/%s.vmdk" % (self._tmp_folder, random_name)) - dest_vmdk_data_file_path = vm_util.build_datastore_path(datastore_name, + dest_vmdk_data_file_path = ds_util.build_datastore_path(datastore_name, "%s/%s-flat.vmdk" % (self._tmp_folder, random_name)) dc_info = self.get_datacenter_ref_and_name(ds_ref) @@ -998,7 +964,7 @@ class VMwareVMOps(object): vm_config_pathname = query['config.files.vmPathName'] datastore_name = None if vm_config_pathname: - _ds_path = vm_util.split_datastore_path(vm_config_pathname) + _ds_path = ds_util.split_datastore_path(vm_config_pathname) datastore_name, vmx_file_path = _ds_path # Power off the VM if it is in PoweredOn state. if pwr_state == "poweredOn": @@ -1022,7 +988,7 @@ class VMwareVMOps(object): # the datastore. if destroy_disks and datastore_name: try: - dir_ds_compliant_path = vm_util.build_datastore_path( + dir_ds_compliant_path = ds_util.build_datastore_path( datastore_name, os.path.dirname(vmx_file_path)) LOG.debug(_("Deleting contents of the VM from " @@ -1032,14 +998,9 @@ class VMwareVMOps(object): ds_ref_ret = query['datastore'] ds_ref = ds_ref_ret.ManagedObjectReference[0] dc_info = self.get_datacenter_ref_and_name(ds_ref) - vim = self._session._get_vim() - delete_task = self._session._call_method( - vim, - "DeleteDatastoreFile_Task", - vim.get_service_content().fileManager, - name=dir_ds_compliant_path, - datacenter=dc_info.ref) - self._session._wait_for_task(delete_task) + ds_util.file_delete(self._session, + dir_ds_compliant_path, + dc_info.ref) LOG.debug(_("Deleted contents of the VM from " "datastore %(datastore_name)s") % {'datastore_name': datastore_name}, @@ -1534,6 +1495,15 @@ class VMwareVMOps(object): "port - %(port)s") % {'port': port}, instance=instance) + def _get_ds_browser(self, ds_ref): + ds_browser = self._datastore_browser_mapping.get(ds_ref) + if not ds_browser: + ds_browser = self._session._call_method( + vim_util, "get_dynamic_property", ds_ref, "Datastore", + "browser") + self._datastore_browser_mapping[ds_ref] = ds_browser + return ds_browser + def get_datacenter_ref_and_name(self, ds_ref): """Get the datacenter name and the reference.""" map = self._datastore_dc_mapping.get(ds_ref.value) @@ -1567,87 +1537,46 @@ class VMwareVMOps(object): vm_folder_ref = dc_objs.objects[0].propSet[0].val return vm_folder_ref - def _path_exists(self, ds_browser, ds_path): - """Check if the path exists on the datastore.""" - search_task = self._session._call_method(self._session._get_vim(), - "SearchDatastore_Task", - ds_browser, - datastorePath=ds_path) - # Wait till the state changes from queued or running. - # If an error state is returned, it means that the path doesn't exist. - while True: - task_info = self._session._call_method(vim_util, - "get_dynamic_property", - search_task, "Task", "info") - if task_info.state in ['queued', 'running']: - time.sleep(2) - continue - break - if task_info.state == "error": - return False - return True + def _create_folder_if_missing(self, ds_name, ds_ref, folder): + """Create a folder if it does not exist. - def _file_exists(self, ds_browser, ds_path, file_name): - """Check if the path and file exists on the datastore.""" - client_factory = self._session._get_vim().client.factory - search_spec = vm_util.search_datastore_spec(client_factory, file_name) - search_task = self._session._call_method(self._session._get_vim(), - "SearchDatastore_Task", - ds_browser, - datastorePath=ds_path, - searchSpec=search_spec) - # Wait till the state changes from queued or running. - # If an error state is returned, it means that the path doesn't exist. - while True: - task_info = self._session._call_method(vim_util, - "get_dynamic_property", - search_task, "Task", "info") - if task_info.state in ['queued', 'running']: - time.sleep(2) - continue - break - if task_info.state == "error": - return False - - file_exists = (getattr(task_info.result, 'file', False) and - task_info.result.file[0].path == file_name) - return file_exists - - def _mkdir(self, ds_path, ds_ref): - """Creates a directory at the path specified. If it is just "NAME", - then a directory with this name is created at the topmost level of the - DataStore. + Currently there are two folder that are required on the datastore + - base folder - the folder to store cached images + - temp folder - the folder used for snapshot management and + image uploading + This method is aimed to be used for the management of those + folders to ensure that they are created if they are missing. + The ds_util method mkdir will be used to check if the folder + exists. If this throws and exception 'FileAlreadyExistsException' + then the folder already exists on the datastore. """ - LOG.debug(_("Creating directory with path %s") % ds_path) + path = ds_util.build_datastore_path(ds_name, folder) dc_info = self.get_datacenter_ref_and_name(ds_ref) - self._session._call_method(self._session._get_vim(), "MakeDirectory", - self._session._get_vim().get_service_content().fileManager, - name=ds_path, datacenter=dc_info.ref, - createParentDirectories=True) - LOG.debug(_("Created directory with path %s") % ds_path) - - def create_cache_folder(self, ds_name, ds_ref): try: - path = vm_util.build_datastore_path(ds_name, self._base_folder) - self._mkdir(path, ds_ref) + ds_util.mkdir(self._session, path, dc_info.ref) + LOG.debug(_("Folder %s created."), path) except error_util.FileAlreadyExistsException: - # NOTE(hartsocks): if the temp folder already exists, that - # just means the temp folder was prepped by another process. + # NOTE(hartsocks): if the folder already exists, that + # just means the folder was prepped by another process. pass + def check_cache_folder(self, ds_name, ds_ref): + """Check that the cache folder exists.""" + self._create_folder_if_missing(ds_name, ds_ref, self._base_folder) + + def check_temp_folder(self, ds_name, ds_ref): + """Check that the temp folder exists.""" + self._create_folder_if_missing(ds_name, ds_ref, self._tmp_folder) + def _check_if_folder_file_exists(self, ds_ref, ds_name, folder_name, file_name): - # Ensure that the cache folder exists - self.create_cache_folder(ds_name, ds_ref) - - ds_browser = self._session._call_method( - vim_util, "get_dynamic_property", ds_ref, "Datastore", "browser") - # Check if the folder exists or not. If not, create one + self.check_cache_folder(ds_name, ds_ref) + ds_browser = self._get_ds_browser(ds_ref) # Check if the file exists or not. - folder_path = vm_util.build_datastore_path(ds_name, folder_name) - file_exists = self._file_exists(ds_browser, folder_path, file_name) - + folder_path = ds_util.build_datastore_path(ds_name, folder_name) + file_exists = ds_util.file_exists(self._session, ds_browser, + folder_path, file_name) return file_exists def inject_network_info(self, instance, network_info):