Fixes Bug "destroy_vm fails with HyperVException"

When trying to delete an instance that has a pending operation,
i.e. creating a snapshot, Hyper-V will raise the following excention:

HyperVException: Operation failed with return value: 32775

Code 32775 means "Invalid state for this operation". This means
that the delete operation cannot be performed while there is
another operation pending.

This patch fixes this problem by requesting all the instance jobs
and killing them right before the delete operation.

Co-Authored-By: Adelina Tuvenie <atuvenie@cloudbasesolutions.com>

Closes Bug: #1461970

Change-Id: I0e3fca20981e1080814224b8cee7cea5b8c6a53f

os_win/tests/utils/test_jobutils.py

@@ -77,12 +77,47 @@ class JobUtilsTestCase(base.BaseTestCase):
                           self.jobutils._wait_for_job,
                           self._FAKE_JOB_PATH)
 
+    def test_wait_for_job_killed(self):
+        mockjob = self._prepare_wait_for_job(constants.JOB_STATE_KILLED)
+        job = self.jobutils._wait_for_job(self._FAKE_JOB_PATH)
+        self.assertEqual(mockjob, job)
+
     def test_wait_for_job_ok(self):
         mock_job = self._prepare_wait_for_job(
             constants.WMI_JOB_STATE_COMPLETED)
         job = self.jobutils._wait_for_job(self._FAKE_JOB_PATH)
         self.assertEqual(mock_job, job)
 
+    def test_stop_jobs(self):
+        mock_job1 = mock.MagicMock(Cancellable=True)
+        mock_job2 = mock.MagicMock(Cancellable=True)
+        mock_job3 = mock.MagicMock(Cancellable=True)
+        mock_job1.JobState = 2
+        mock_job2.JobState = 3
+        mock_job3.JobState = constants.JOB_STATE_KILLED
+
+        mock_vm = mock.MagicMock()
+        mock_vm_jobs = [mock_job1, mock_job2, mock_job3]
+        mock_vm.associators.return_value = mock_vm_jobs
+
+        self.jobutils.stop_jobs(mock_vm)
+
+        mock_job1.RequestStateChange.assert_called_once_with(
+            self.jobutils._KILL_JOB_STATE_CHANGE_REQUEST)
+        mock_job2.RequestStateChange.assert_called_once_with(
+            self.jobutils._KILL_JOB_STATE_CHANGE_REQUEST)
+        self.assertFalse(mock_job3.RequestStateChange.called)
+
+    def test_is_job_completed_true(self):
+        job = mock.MagicMock(JobState=constants.JOB_STATE_COMPLETED)
+
+        self.assertTrue(self.jobutils._is_job_completed(job))
+
+    def test_is_job_completed_false(self):
+        job = mock.MagicMock(JobState=constants.WMI_JOB_STATE_RUNNING)
+
+        self.assertFalse(self.jobutils._is_job_completed(job))
+
     def _prepare_wait_for_job(self, state=_FAKE_JOB_STATUS_BAD):
         mock_job = mock.MagicMock()
         mock_job.JobState = state

os_win/tests/utils/test_vmutils.py

@@ -746,3 +746,10 @@ class VMUtilsTestCase(base.BaseTestCase):
                 fields=[self._vmutils._VM_ENABLED_STATE_PROP])
 
             self.assertEqual(watcher.return_value, listener)
+
+    def test_stop_vm_jobs(self):
+        mock_vm = self._lookup_vm()
+
+        self._vmutils.stop_vm_jobs(mock.sentinel.vm_name)
+
+        self._vmutils._jobutils.stop_jobs.assert_called_once_with(mock_vm)

os_win/utils/constants.py

@@ -67,6 +67,10 @@ IMAGE_PROP_VM_GEN_2 = "hyperv-gen2"
 VM_GEN_1 = 1
 VM_GEN_2 = 2
 
+JOB_STATE_COMPLETED = 7
+JOB_STATE_TERMINATED = 8
+JOB_STATE_KILLED = 9
+JOB_STATE_COMPLETED_WITH_WARNINGS = 32768
 
 # Special vlan_id value in ovs_vlan_allocations table indicating flat network
 FLAT_VLAN_ID = -1

os_win/utils/jobutils.py

@@ -37,6 +37,15 @@ class JobUtils(object):
 
     _WMI_NAMESPACE = '//%s/root/virtualization'
 
+    _CONCRETE_JOB_CLASS = "Msvm_ConcreteJob"
+
+    _KILL_JOB_STATE_CHANGE_REQUEST = 5
+
+    _completed_job_states = [constants.JOB_STATE_COMPLETED,
+                             constants.JOB_STATE_TERMINATED,
+                             constants.JOB_STATE_KILLED,
+                             constants.JOB_STATE_COMPLETED_WITH_WARNINGS]
+
     def __init__(self, host='.'):
         if sys.platform == 'win32':
             self._init_hyperv_wmi_conn(host)
@@ -61,6 +70,11 @@ class JobUtils(object):
         while job.JobState == constants.WMI_JOB_STATE_RUNNING:
             time.sleep(0.1)
             job = wmi.WMI(moniker=job_wmi_path)
+
+        if job.JobState == constants.JOB_STATE_KILLED:
+            LOG.debug("WMI job killed with status %s.", job.JobState)
+            return job
+
         if job.JobState != constants.WMI_JOB_STATE_COMPLETED:
             job_state = job.JobState
             if job.path().Class == "Msvm_ConcreteJob":
@@ -93,6 +107,18 @@ class JobUtils(object):
                   {'desc': desc, 'elap': elap})
         return job
 
+    def stop_jobs(self, element):
+        jobs = element.associators(wmi_result_class=self._CONCRETE_JOB_CLASS)
+
+        for job in jobs:
+            if job and job.Cancellable and not self._is_job_completed(job):
+                job.RequestStateChange(self._KILL_JOB_STATE_CHANGE_REQUEST)
+
+        return jobs
+
+    def _is_job_completed(self, job):
+        return job.JobState in self._completed_job_states
+
     def add_virt_resource(self, virt_resource, parent):
         """Adds a new resource to the VM."""
         vs_man_svc = self._conn.Msvm_VirtualSystemManagementService()[0]

os_win/utils/vmutils.py

@@ -729,3 +729,7 @@ class VMUtils(object):
     def get_vm_power_state(self, vm_enabled_state):
         return self._enabled_states_map.get(vm_enabled_state,
                                             constants.HYPERV_VM_STATE_OTHER)
+
+    def stop_vm_jobs(self, vm_name):
+        vm = self._lookup_vm_check(vm_name)
+        self._jobutils.stop_jobs(vm)