VMware: add support for graceful shutdown of instances

Change-Id: I40643e9d358be89c87a0311b1c1fd7718ec75361
Closes-Bug: #1377781
Co-Authored-By: David Rabel <rabel@b1-systems.de>

nova/tests/unit/virt/vmwareapi/test_vmops.py

@@ -13,6 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import time
+
 import mock
 from oslo_serialization import jsonutils
 from oslo_utils import units
@@ -516,6 +518,108 @@ class VMwareVMOpsTestCase(test.NoDBTestCase):
             _volumeops.detach_disk_from_vm.assert_called_once_with(
                 vm_ref, self._instance, mock.ANY, destroy_disk=True)
 
+    @mock.patch.object(time, 'sleep')
+    def _test_clean_shutdown(self, mock_sleep,
+                             timeout, retry_interval,
+                             returns_on, returns_off,
+                             vmware_tools_status,
+                             succeeds):
+        """Test the _clean_shutdown method
+
+        :param timeout: timeout before soft shutdown is considered a fail
+        :param retry_interval: time between rechecking instance power state
+        :param returns_on: how often the instance is reported as poweredOn
+        :param returns_off: how often the instance is reported as poweredOff
+        :param vmware_tools_status: Status of vmware tools
+        :param succeeds: the expected result
+        """
+        instance = self._instance
+        vm_ref = mock.Mock()
+        return_props = []
+        expected_methods = ['get_object_properties_dict']
+        props_on = {'runtime.powerState': 'poweredOn',
+                   'summary.guest.toolsStatus': vmware_tools_status,
+                   'summary.guest.toolsRunningStatus': 'guestToolsRunning'}
+        props_off = {'runtime.powerState': 'poweredOff',
+                    'summary.guest.toolsStatus': vmware_tools_status,
+                    'summary.guest.toolsRunningStatus': 'guestToolsRunning'}
+
+        # initialize expected instance methods and returned properties
+        if vmware_tools_status == "toolsOk":
+            if returns_on > 0:
+                expected_methods.append('ShutdownGuest')
+                for x in range(returns_on + 1):
+                    return_props.append(props_on)
+                for x in range(returns_on):
+                    expected_methods.append('get_object_properties_dict')
+            for x in range(returns_off):
+                return_props.append(props_off)
+                if returns_on > 0:
+                    expected_methods.append('get_object_properties_dict')
+        else:
+            return_props.append(props_off)
+
+        def fake_call_method(module, method, *args, **kwargs):
+            expected_method = expected_methods.pop(0)
+            self.assertEqual(expected_method, method)
+            if expected_method == 'get_object_properties_dict':
+                props = return_props.pop(0)
+                return props
+            elif expected_method == 'ShutdownGuest':
+                return
+
+        with test.nested(
+                mock.patch.object(vm_util, 'get_vm_ref', return_value=vm_ref),
+                mock.patch.object(self._session, '_call_method',
+                                  side_effect=fake_call_method)
+        ) as (mock_get_vm_ref, mock_call_method):
+            result = self._vmops._clean_shutdown(instance, timeout,
+                                                 retry_interval)
+
+        self.assertEqual(succeeds, result)
+        mock_get_vm_ref.assert_called_once_with(self._session,
+                                                self._instance)
+
+    def test_clean_shutdown_first_time(self):
+        self._test_clean_shutdown(timeout=10,
+                                  retry_interval=3,
+                                  returns_on=1,
+                                  returns_off=1,
+                                  vmware_tools_status="toolsOk",
+                                  succeeds=True)
+
+    def test_clean_shutdown_second_time(self):
+        self._test_clean_shutdown(timeout=10,
+                                  retry_interval=3,
+                                  returns_on=2,
+                                  returns_off=1,
+                                  vmware_tools_status="toolsOk",
+                                  succeeds=True)
+
+    def test_clean_shutdown_timeout(self):
+        self._test_clean_shutdown(timeout=10,
+                                  retry_interval=3,
+                                  returns_on=4,
+                                  returns_off=0,
+                                  vmware_tools_status="toolsOk",
+                                  succeeds=False)
+
+    def test_clean_shutdown_already_off(self):
+        self._test_clean_shutdown(timeout=10,
+                                  retry_interval=3,
+                                  returns_on=0,
+                                  returns_off=1,
+                                  vmware_tools_status="toolsOk",
+                                  succeeds=False)
+
+    def test_clean_shutdown_no_vwaretools(self):
+        self._test_clean_shutdown(timeout=10,
+                                  retry_interval=3,
+                                  returns_on=1,
+                                  returns_off=0,
+                                  vmware_tools_status="toolsNotOk",
+                                  succeeds=False)
+
     def _test_finish_migration(self, power_on=True, resize_instance=False):
         with test.nested(
                 mock.patch.object(self._vmops,

nova/virt/vmwareapi/driver.py

@@ -427,8 +427,7 @@ class VMwareVCDriver(driver.ComputeDriver):
 
     def power_off(self, instance, timeout=0, retry_interval=0):
         """Power off the specified instance."""
-        # TODO(PhilDay): Add support for timeout (clean shutdown)
-        self._vmops.power_off(instance)
+        self._vmops.power_off(instance, timeout, retry_interval)
 
     def power_on(self, context, instance, network_info,
                  block_device_info=None):

nova/virt/vmwareapi/vmops.py

@@ -1244,13 +1244,80 @@ class VMwareVMOps(object):
         if power_on:
             vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref)
 
-    def power_off(self, instance):
+    def power_off(self, instance, timeout=0, retry_interval=0):
         """Power off the specified instance.
 
         :param instance: nova.objects.instance.Instance
+        :param timeout: How long to wait in seconds for the instance to
+                        shutdown
+        :param retry_interval: Interval to check if instance is already
+                               shutdown in seconds.
         """
+        if timeout and self._clean_shutdown(instance,
+                                            timeout,
+                                            retry_interval):
+            return
+
         vm_util.power_off_instance(self._session, instance)
 
+    def _clean_shutdown(self, instance, timeout, retry_interval):
+        """Perform a soft shutdown on the VM.
+        :param instance: nova.objects.instance.Instance
+        :param timeout: How long to wait in seconds for the instance to
+                        shutdown
+        :param retry_interval: Interval to check if instance is already
+                               shutdown in seconds.
+           :return: True if the instance was shutdown within time limit,
+                    False otherwise.
+        """
+        LOG.debug("Performing Soft shutdown on instance",
+                 instance=instance)
+        vm_ref = vm_util.get_vm_ref(self._session, instance)
+
+        props = self._get_instance_props(vm_ref)
+
+        if props.get("runtime.powerState") != "poweredOn":
+            LOG.debug("Instance not in poweredOn state.",
+                      instance=instance)
+            return False
+
+        if ((props.get("summary.guest.toolsStatus") == "toolsOk") and
+            (props.get("summary.guest.toolsRunningStatus") ==
+             "guestToolsRunning")):
+
+            LOG.debug("Soft shutdown instance, timeout: %d",
+                     timeout, instance=instance)
+            self._session._call_method(self._session.vim,
+                                       "ShutdownGuest",
+                                       vm_ref)
+
+            while timeout > 0:
+                wait_time = min(retry_interval, timeout)
+                props = self._get_instance_props(vm_ref)
+
+                if props.get("runtime.powerState") == "poweredOff":
+                    LOG.info("Soft shutdown succeeded.",
+                             instance=instance)
+                    return True
+
+                time.sleep(wait_time)
+                timeout -= retry_interval
+
+            LOG.warning("Timed out while waiting for soft shutdown.",
+                        instance=instance)
+        else:
+            LOG.debug("VMware Tools not running", instance=instance)
+
+        return False
+
+    def _get_instance_props(self, vm_ref):
+        lst_properties = ["summary.guest.toolsStatus",
+                          "runtime.powerState",
+                          "summary.guest.toolsRunningStatus"]
+        return self._session._call_method(vutil,
+                                          "get_object_properties_dict",
+                                          vm_ref, lst_properties)
+
     def power_on(self, instance):
         vm_util.power_on_instance(self._session, instance)
 

releasenotes/notes/bug-1377781-c91d5319862bb9d8.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Add support for graceful shutdown of VMware instances. The timeout parameter of the power_off
+    method is now considered by the VMware driver. If you specify a timeout greater than 0, the
+    driver calls the appropriate soft shutdown method of the VMware API and only forces a hard
+    shutdown if the soft shutdown did not succeed before the timeout is reached.