x/nova-zvm-virt-driver
Code Issues Proposed changes

Try to do unlock when delete userid failed

Browse Source 
Right now, if delete userid failed due to image lock or device
lock, zvm driver has a looping call to check lock status with 10
minutes timeout.

Normaly, the image or device was locked by disk related actions
which could not be unlocked in short time. So turn to call
image_unlock_dm try to unlock it. Then try to delete the vm again.

Change-Id: I848d042231757d8673700e3db2b46f8fc9757596
This commit is contained in:
Huang Rui
2017-03-15 16:02:08 +08:00
parent 4313136156
commit 576861d24c
2 changed files with 68 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
nova/tests/unit/virt/zvm/test_zvm.py
View File

@@ -1969,87 +1969,25 @@ class ZVMInstanceTestCases(ZVMTestCase):
self.assertRaises(exception.ZVMXCATDeployNodeFailed,
self._instance.deploy_node, 'fakeimg', '/fake/file')
def test_delete_userid_is_locked(self):
resp = {'error': [['Return Code: 400\nReason Code: 16\n']]}
self.mox.StubOutWithMock(zvmutils, 'xcat_request')
self.mox.StubOutWithMock(self._instance, '_wait_for_unlock')
zvmutils.xcat_request("DELETE", mox.IgnoreArg()).AndRaise(
exception.ZVMXCATInternalError(msg=str(resp)))
self._instance._wait_for_unlock('fakehcp')
zvmutils.xcat_request("DELETE", mox.IgnoreArg())
self.mox.ReplayAll()
@mock.patch('nova.virt.zvm.instance.ZVMInstance.unlock_userid')
@mock.patch('nova.virt.zvm.instance.ZVMInstance._delete_userid')
def test_delete_userid_is_locked(self, delete_uid, unlock_uid):
delete_uid.side_effect = [exception.ZVMXCATInternalError(
'Return Code: 400\nReason Code: 12\n'),
None]
self._instance.delete_userid('fakehcp', {})
self.mox.VerifyAll()
def test_delete_userid_is_locked_and_notexist(self):
resp = {'error': [['Return Code: 400\nReason Code: 16\n']]}
self.mox.StubOutWithMock(self._instance, 'delete_xcat_node')
self.mox.StubOutWithMock(zvmutils, 'xcat_request')
self.mox.StubOutWithMock(self._instance, '_wait_for_unlock')
zvmutils.xcat_request("DELETE", mox.IgnoreArg()).AndRaise(
exception.ZVMXCATInternalError(msg=str(resp)))
self._instance._wait_for_unlock('fakehcp')
resp = {'error': [['Return Code: 400\nReason Code: 4\n']]}
zvmutils.xcat_request("DELETE", mox.IgnoreArg()).AndRaise(
exception.ZVMXCATInternalError(msg=str(resp)))
self._instance.delete_xcat_node()
self.mox.ReplayAll()
delete_uid.assert_called()
unlock_uid.assert_called_once_with('fakehcp')
@mock.patch('nova.virt.zvm.instance.ZVMInstance.unlock_devices')
@mock.patch('nova.virt.zvm.instance.ZVMInstance._delete_userid')
def test_delete_userid_device_is_locked(self, delete_uid, unlock_dev):
delete_uid.side_effect = [exception.ZVMXCATInternalError(
'Return Code: 408\nReason Code: 12\n'),
None]
self._instance.delete_userid('fakehcp', {})
self.mox.VerifyAll()
def test_delete_userid_400012(self):
resp = {'error': [['Return Code: 400\nReason Code: 12\n']]}
self.mox.StubOutWithMock(zvmutils, 'xcat_request')
self.mox.StubOutWithMock(self._instance, '_wait_for_unlock')
zvmutils.xcat_request("DELETE", mox.IgnoreArg()).AndRaise(
exception.ZVMXCATInternalError(msg=str(resp)))
self._instance._wait_for_unlock('fakehcp')
zvmutils.xcat_request("DELETE", mox.IgnoreArg())
self.mox.ReplayAll()
self._instance.delete_userid('fakehcp', {})
self.mox.VerifyAll()
def test_is_locked_true(self):
resp = {'data': [['os000001: os000001 is locked']]}
self.mox.StubOutWithMock(zvmutils, 'xdsh')
execstr = "/opt/zhcp/bin/smcli Image_Lock_Query_DM -T os000001"
zvmutils.xdsh('fakehcp', execstr).AndReturn(resp)
self.mox.ReplayAll()
locked = self._instance.is_locked('fakehcp')
self.mox.VerifyAll()
self.assertTrue(locked)
def test_is_locked_false(self):
resp = {'data': [['os000001: os000001 is Unlocked...']]}
self.mox.StubOutWithMock(zvmutils, 'xdsh')
execstr = "/opt/zhcp/bin/smcli Image_Lock_Query_DM -T os000001"
zvmutils.xdsh('fakehcp', execstr).AndReturn(resp)
self.mox.ReplayAll()
locked = self._instance.is_locked('fakehcp')
self.mox.VerifyAll()
self.assertFalse(locked)
def test_wait_for_unlock(self):
self.mox.StubOutWithMock(self._instance, 'is_locked')
self._instance.is_locked('fakehcp').AndReturn(True)
self._instance.is_locked('fakehcp').AndReturn(False)
self.mox.ReplayAll()
self._instance._wait_for_unlock('fakehcp', 1)
self.mox.VerifyAll()
delete_uid.assert_called()
unlock_dev.assert_called_once_with('fakehcp')
def test_modify_storage_format(self):
mem = self._instance._modify_storage_format('0')
@@ -2166,6 +2104,24 @@ class ZVMInstanceTestCases(ZVMTestCase):
self.assertEqual(0, inst_info.cpu_time_ns)
self.assertEqual(1048576, inst_info.max_mem_kb)
@mock.patch('nova.virt.zvm.utils.xdsh')
@mock.patch('nova.virt.zvm.instance.ZVMInstance.get_userid')
def test_unlock_devices(self, get_uid, xdsh):
get_uid.return_value = 'fakeuid'
xdshv1 = {'data': [['Locked type: DEVICE\nDevice address: 0100\n'
'Device locked by: fake\nDevice address: 0101\n'
'Device locked by: fake']]}
xdsh.side_effect = [xdshv1, None, None]
self._instance.unlock_devices('fakezhcp')
get_uid.assert_called_with()
xdsh.assert_any_call('fakezhcp',
'/opt/zhcp/bin/smcli Image_Lock_Query_DM -T fakeuid')
xdsh.assert_any_call('fakezhcp',
'/opt/zhcp/bin/smcli Image_Unlock_DM -T fakeuid -v 0100')
xdsh.assert_any_call('fakezhcp',
'/opt/zhcp/bin/smcli Image_Unlock_DM -T fakeuid -v 0101')
class ZVMXCATConnectionTestCases(test.TestCase):
"""Test cases for xCAT connection."""

61
nova/virt/zvm/instance.py
View File

@@ -14,14 +14,11 @@
import binascii
import datetime
import six
import time
from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import loopingcall
from oslo_utils import timeutils
from nova.compute import power_state
from nova import exception as nova_exception
@@ -460,29 +457,35 @@ class ZVMInstance(object):
url = self._xcat_url.chvm('/' + self._name)
zvmutils.xcat_request("PUT", url, body)
def is_locked(self, zhcp_node):
cmd = "/opt/zhcp/bin/smcli Image_Lock_Query_DM -T %s" % self._name
def get_userid(self):
return zvmutils.get_userid(self._name)
def unlock_userid(self, zhcp_node):
_uid = self.get_userid()
cmd = "/opt/zhcp/bin/smcli Image_Unlock_DM -T %s" % _uid
zvmutils.xdsh(zhcp_node, cmd)
def unlock_devices(self, zhcp_node):
_uid = self.get_userid()
cmd = "/opt/zhcp/bin/smcli Image_Lock_Query_DM -T %s" % _uid
resp = zvmutils.xdsh(zhcp_node, cmd)
with zvmutils.expect_invalid_xcat_resp_data(resp):
resp_str = resp['data'][0][0]
return "is Unlocked..." not in str(resp)
if resp_str.__contains__("is Unlocked..."):
# unlocked automatically, do nothing
return
def _wait_for_unlock(self, zhcp_node, interval=60, timeout=300):
LOG.debug("Waiting for unlock instance %s", self._name)
def _unlock_device(vdev):
cmd = ("/opt/zhcp/bin/smcli Image_Unlock_DM -T %(uid)s -v %(vdev)s"
% {'uid': _uid, 'vdev': vdev})
zvmutils.xdsh(zhcp_node, cmd)
def _wait_unlock(expiration):
if timeutils.utcnow() > expiration:
LOG.debug("Waiting for unlock instance %s timeout", self._name)
raise loopingcall.LoopingCallDone()
if not self.is_locked(zhcp_node):
LOG.debug("Instance %s is unlocked", self._name)
raise loopingcall.LoopingCallDone()
expiration = timeutils.utcnow() + datetime.timedelta(seconds=timeout)
timer = loopingcall.FixedIntervalLoopingCall(_wait_unlock,
expiration)
timer.start(interval=interval).wait()
resp_list = resp_str.split('\n')
for s in resp_list:
if s.__contains__('Device address:'):
vdev = s.rpartition(':')[2].strip()
_unlock_device(vdev)
def _delete_userid(self, url):
try:
@@ -511,15 +514,19 @@ class ZVMInstance(object):
except exception.ZVMXCATInternalError as err:
emsg = err.format_message()
if (emsg.__contains__("Return Code: 400") and
(emsg.__contains__("Reason Code: 16") or
emsg.__contains__("Reason Code: 12"))):
# The vm or vm device was locked. Unlock before deleting
self._wait_for_unlock(zhcp_node)
self._delete_userid(url)
emsg.__contains__("Reason Code: 12")):
# The vm was locked. Unlock before deleting
self.unlock_userid(zhcp_node)
elif (emsg.__contains__("Return Code: 408") and
emsg.__contains__("Reason Code: 12")):
# The vm device was locked. Unlock the device before deleting
self.unlock_devices(zhcp_node)
else:
LOG.debug("exception not able to handle in delete_userid "
"%s", self._name)
raise err
# delete the vm after unlock
self._delete_userid(url)
except exception.ZVMXCATRequestFailed as err:
emsg = err.format_message()
if (emsg.__contains__("Invalid nodes and/or groups") and