Try to do unlock when delete userid failed

Right now, if delete userid failed due to image lock or device
lock, zvm driver has a looping call to check lock status with 10
minutes timeout.

Normaly, the image or device was locked by disk related actions
which could not be unlocked in short time. So turn to call
image_unlock_dm try to unlock it. Then try to delete the vm again.

Change-Id: I848d042231757d8673700e3db2b46f8fc9757596

nova/tests/unit/virt/zvm/test_zvm.py

@@ -1969,87 +1969,25 @@ class ZVMInstanceTestCases(ZVMTestCase):
         self.assertRaises(exception.ZVMXCATDeployNodeFailed,
                           self._instance.deploy_node, 'fakeimg', '/fake/file')
 
-    def test_delete_userid_is_locked(self):
+    @mock.patch('nova.virt.zvm.instance.ZVMInstance.unlock_userid')
-        resp = {'error': [['Return Code: 400\nReason Code: 16\n']]}
+    @mock.patch('nova.virt.zvm.instance.ZVMInstance._delete_userid')
-
+    def test_delete_userid_is_locked(self, delete_uid, unlock_uid):
-        self.mox.StubOutWithMock(zvmutils, 'xcat_request')
+        delete_uid.side_effect = [exception.ZVMXCATInternalError(
-        self.mox.StubOutWithMock(self._instance, '_wait_for_unlock')
+                                    'Return Code: 400\nReason Code: 12\n'),
-        zvmutils.xcat_request("DELETE", mox.IgnoreArg()).AndRaise(
+                                  None]
-            exception.ZVMXCATInternalError(msg=str(resp)))
-        self._instance._wait_for_unlock('fakehcp')
-        zvmutils.xcat_request("DELETE", mox.IgnoreArg())
-        self.mox.ReplayAll()
-
         self._instance.delete_userid('fakehcp', {})
-        self.mox.VerifyAll()
+        delete_uid.assert_called()
-
+        unlock_uid.assert_called_once_with('fakehcp')
-    def test_delete_userid_is_locked_and_notexist(self):
-        resp = {'error': [['Return Code: 400\nReason Code: 16\n']]}
-
-        self.mox.StubOutWithMock(self._instance, 'delete_xcat_node')
-        self.mox.StubOutWithMock(zvmutils, 'xcat_request')
-        self.mox.StubOutWithMock(self._instance, '_wait_for_unlock')
-        zvmutils.xcat_request("DELETE", mox.IgnoreArg()).AndRaise(
-            exception.ZVMXCATInternalError(msg=str(resp)))
-        self._instance._wait_for_unlock('fakehcp')
-
-        resp = {'error': [['Return Code: 400\nReason Code: 4\n']]}
-        zvmutils.xcat_request("DELETE", mox.IgnoreArg()).AndRaise(
-            exception.ZVMXCATInternalError(msg=str(resp)))
-        self._instance.delete_xcat_node()
-        self.mox.ReplayAll()
 
+    @mock.patch('nova.virt.zvm.instance.ZVMInstance.unlock_devices')
+    @mock.patch('nova.virt.zvm.instance.ZVMInstance._delete_userid')
+    def test_delete_userid_device_is_locked(self, delete_uid, unlock_dev):
+        delete_uid.side_effect = [exception.ZVMXCATInternalError(
+                                    'Return Code: 408\nReason Code: 12\n'),
+                                  None]
         self._instance.delete_userid('fakehcp', {})
-        self.mox.VerifyAll()
+        delete_uid.assert_called()
-
+        unlock_dev.assert_called_once_with('fakehcp')
-    def test_delete_userid_400012(self):
-        resp = {'error': [['Return Code: 400\nReason Code: 12\n']]}
-
-        self.mox.StubOutWithMock(zvmutils, 'xcat_request')
-        self.mox.StubOutWithMock(self._instance, '_wait_for_unlock')
-        zvmutils.xcat_request("DELETE", mox.IgnoreArg()).AndRaise(
-            exception.ZVMXCATInternalError(msg=str(resp)))
-        self._instance._wait_for_unlock('fakehcp')
-        zvmutils.xcat_request("DELETE", mox.IgnoreArg())
-        self.mox.ReplayAll()
-
-        self._instance.delete_userid('fakehcp', {})
-        self.mox.VerifyAll()
-
-    def test_is_locked_true(self):
-        resp = {'data': [['os000001: os000001 is locked']]}
-
-        self.mox.StubOutWithMock(zvmutils, 'xdsh')
-        execstr = "/opt/zhcp/bin/smcli Image_Lock_Query_DM -T os000001"
-        zvmutils.xdsh('fakehcp', execstr).AndReturn(resp)
-        self.mox.ReplayAll()
-
-        locked = self._instance.is_locked('fakehcp')
-        self.mox.VerifyAll()
-
-        self.assertTrue(locked)
-
-    def test_is_locked_false(self):
-        resp = {'data': [['os000001: os000001 is Unlocked...']]}
-
-        self.mox.StubOutWithMock(zvmutils, 'xdsh')
-        execstr = "/opt/zhcp/bin/smcli Image_Lock_Query_DM -T os000001"
-        zvmutils.xdsh('fakehcp', execstr).AndReturn(resp)
-        self.mox.ReplayAll()
-
-        locked = self._instance.is_locked('fakehcp')
-        self.mox.VerifyAll()
-
-        self.assertFalse(locked)
-
-    def test_wait_for_unlock(self):
-        self.mox.StubOutWithMock(self._instance, 'is_locked')
-        self._instance.is_locked('fakehcp').AndReturn(True)
-        self._instance.is_locked('fakehcp').AndReturn(False)
-        self.mox.ReplayAll()
-
-        self._instance._wait_for_unlock('fakehcp', 1)
-        self.mox.VerifyAll()
 
     def test_modify_storage_format(self):
         mem = self._instance._modify_storage_format('0')
@@ -2166,6 +2104,24 @@ class ZVMInstanceTestCases(ZVMTestCase):
         self.assertEqual(0, inst_info.cpu_time_ns)
         self.assertEqual(1048576, inst_info.max_mem_kb)
 
+    @mock.patch('nova.virt.zvm.utils.xdsh')
+    @mock.patch('nova.virt.zvm.instance.ZVMInstance.get_userid')
+    def test_unlock_devices(self, get_uid, xdsh):
+        get_uid.return_value = 'fakeuid'
+        xdshv1 = {'data': [['Locked type: DEVICE\nDevice address: 0100\n'
+                            'Device locked by: fake\nDevice address: 0101\n'
+                            'Device locked by: fake']]}
+        xdsh.side_effect = [xdshv1, None, None]
+        self._instance.unlock_devices('fakezhcp')
+
+        get_uid.assert_called_with()
+        xdsh.assert_any_call('fakezhcp',
+            '/opt/zhcp/bin/smcli Image_Lock_Query_DM -T fakeuid')
+        xdsh.assert_any_call('fakezhcp',
+            '/opt/zhcp/bin/smcli Image_Unlock_DM -T fakeuid -v 0100')
+        xdsh.assert_any_call('fakezhcp',
+            '/opt/zhcp/bin/smcli Image_Unlock_DM -T fakeuid -v 0101')
+
 
 class ZVMXCATConnectionTestCases(test.TestCase):
     """Test cases for xCAT connection."""

nova/virt/zvm/instance.py

@@ -14,14 +14,11 @@
 
 
 import binascii
-import datetime
 import six
 import time
 
 from oslo_config import cfg
 from oslo_log import log as logging
-from oslo_service import loopingcall
-from oslo_utils import timeutils
 
 from nova.compute import power_state
 from nova import exception as nova_exception
@@ -460,29 +457,35 @@ class ZVMInstance(object):
         url = self._xcat_url.chvm('/' + self._name)
         zvmutils.xcat_request("PUT", url, body)
 
-    def is_locked(self, zhcp_node):
+    def get_userid(self):
-        cmd = "/opt/zhcp/bin/smcli Image_Lock_Query_DM -T %s" % self._name
+        return zvmutils.get_userid(self._name)
+
+    def unlock_userid(self, zhcp_node):
+        _uid = self.get_userid()
+        cmd = "/opt/zhcp/bin/smcli Image_Unlock_DM -T %s" % _uid
+        zvmutils.xdsh(zhcp_node, cmd)
+
+    def unlock_devices(self, zhcp_node):
+        _uid = self.get_userid()
+        cmd = "/opt/zhcp/bin/smcli Image_Lock_Query_DM -T %s" % _uid
         resp = zvmutils.xdsh(zhcp_node, cmd)
+        with zvmutils.expect_invalid_xcat_resp_data(resp):
+            resp_str = resp['data'][0][0]
 
-        return "is Unlocked..." not in str(resp)
+        if resp_str.__contains__("is Unlocked..."):
+            # unlocked automatically, do nothing
+            return
 
-    def _wait_for_unlock(self, zhcp_node, interval=60, timeout=300):
+        def _unlock_device(vdev):
-        LOG.debug("Waiting for unlock instance %s", self._name)
+            cmd = ("/opt/zhcp/bin/smcli Image_Unlock_DM -T %(uid)s -v %(vdev)s"
+                   % {'uid': _uid, 'vdev': vdev})
+            zvmutils.xdsh(zhcp_node, cmd)
 
-        def _wait_unlock(expiration):
+        resp_list = resp_str.split('\n')
-            if timeutils.utcnow() > expiration:
+        for s in resp_list:
-                LOG.debug("Waiting for unlock instance %s timeout", self._name)
+            if s.__contains__('Device address:'):
-                raise loopingcall.LoopingCallDone()
+                vdev = s.rpartition(':')[2].strip()
-
+                _unlock_device(vdev)
-            if not self.is_locked(zhcp_node):
-                LOG.debug("Instance %s is unlocked", self._name)
-                raise loopingcall.LoopingCallDone()
-
-        expiration = timeutils.utcnow() + datetime.timedelta(seconds=timeout)
-
-        timer = loopingcall.FixedIntervalLoopingCall(_wait_unlock,
-                                                     expiration)
-        timer.start(interval=interval).wait()
 
     def _delete_userid(self, url):
         try:
@@ -511,15 +514,19 @@ class ZVMInstance(object):
         except exception.ZVMXCATInternalError as err:
             emsg = err.format_message()
             if (emsg.__contains__("Return Code: 400") and
-               (emsg.__contains__("Reason Code: 16") or
+                    emsg.__contains__("Reason Code: 12")):
-                emsg.__contains__("Reason Code: 12"))):
+                # The vm was locked. Unlock before deleting
-                # The vm or vm device was locked. Unlock before deleting
+                self.unlock_userid(zhcp_node)
-                self._wait_for_unlock(zhcp_node)
+            elif (emsg.__contains__("Return Code: 408") and
-                self._delete_userid(url)
+                    emsg.__contains__("Reason Code: 12")):
+                # The vm device was locked. Unlock the device before deleting
+                self.unlock_devices(zhcp_node)
             else:
                 LOG.debug("exception not able to handle in delete_userid "
                           "%s", self._name)
                 raise err
+            # delete the vm after unlock
+            self._delete_userid(url)
         except exception.ZVMXCATRequestFailed as err:
             emsg = err.format_message()
             if (emsg.__contains__("Invalid nodes and/or groups") and