Adds ability to reset iDRAC

Adds new function to reset the iDRAC and wait for it
to become operational again.

Change-Id: Ia8dc0b97e02fc5f2c4d39b6b6d90456c1cfc5b7a
Co-Authored-By: Christopher Dearborn <christopher.dearborn@dell.com>
This commit is contained in:
Mark Beierl 2018-07-03 15:19:28 -04:00
parent a300f85654
commit 88023841ef
7 changed files with 469 additions and 0 deletions

View File

@ -16,6 +16,7 @@ Wrapper for pywsman.Client
"""
import logging
import subprocess
import time
from dracclient import constants
@ -241,6 +242,103 @@ class DRACClient(object):
"""
return self._idrac_cfg.set_idrac_settings(settings, idrac_fqdd)
def reset_idrac(self, force=False, wait=False,
ready_wait_time=30):
"""Resets the iDRAC and optionally block until reset is complete.
:param force: does a force reset when True and a graceful reset when
False
:param wait: returns immediately after reset if False, or waits
for the iDRAC to return to operational state if True
:param ready_wait_time: the amount of time in seconds to wait after
the reset before starting to check on the iDRAC's status
:returns: True on success, raises exception on failure
:raises: WSManRequestFailure on request failures
:raises: WSManInvalidResponse when receiving invalid response
:raises: DRACOperationFailed on failure to reset iDRAC
"""
return_value = self._idrac_cfg.reset_idrac(force)
if not wait and return_value:
return return_value
if not return_value:
raise exceptions.DRACOperationFailed(
drac_messages="Failed to reset iDRAC")
LOG.debug("iDRAC was reset, waiting for return to operational state")
state_reached = self._wait_for_host_state(
self.client.host,
alive=False,
ping_count=3,
retries=24)
if not state_reached:
raise exceptions.DRACOperationFailed(
drac_messages="Timed out waiting for the %s iDRAC to become "
"not pingable" % self.client.host)
LOG.info("The iDRAC has become not pingable")
state_reached = self._wait_for_host_state(
self.client.host,
alive=True,
ping_count=3,
retries=24)
if not state_reached:
raise exceptions.DRACOperationFailed(
drac_messages="Timed out waiting for the %s iDRAC to become "
"pingable" % self.client.host)
LOG.info("The iDRAC has become pingable")
LOG.info("Waiting for the iDRAC to become ready")
time.sleep(ready_wait_time)
self.client.wait_until_idrac_is_ready()
def _ping_host(self, host):
response = subprocess.call(
"ping -c 1 {} 2>&1 1>/dev/null".format(host), shell=True)
return (response == 0)
def _wait_for_host_state(self,
host,
alive=True,
ping_count=3,
retries=24):
if alive:
ping_type = "pingable"
else:
ping_type = "not pingable"
LOG.info("Waiting for the iDRAC to become %s", ping_type)
response_count = 0
state_reached = False
while retries > 0 and not state_reached:
response = self._ping_host(host)
retries -= 1
if response == alive:
response_count += 1
LOG.debug("The iDRAC is %s, count=%s",
ping_type,
response_count)
if response_count == ping_count:
LOG.debug("Reached specified ping count")
state_reached = True
else:
response_count = 0
if alive:
LOG.debug("The iDRAC is still not pingable")
else:
LOG.debug("The iDRAC is still pingable")
time.sleep(10)
return state_reached
def commit_pending_idrac_changes(
self,
idrac_fqdd=IDRAC_FQDD,

View File

@ -321,6 +321,33 @@ class iDRACCardConfiguration(object):
idrac_fqdd,
name_formatter=_name_formatter)
def reset_idrac(self, force=False):
"""Resets the iDRAC
:param force: does a force reset when True and a graceful reset when
False.
:returns: True on success and False on failure.
:raises: WSManRequestFailure on request failures
:raises: WSManInvalidResponse when receiving invalid response
"""
selectors = {'CreationClassName': "DCIM_iDRACCardService",
'Name': "DCIM:iDRACCardService",
'SystemCreationClassName': 'DCIM_ComputerSystem',
'SystemName': 'DCIM:ComputerSystem'}
properties = {'Force': "1" if force else "0"}
doc = self.client.invoke(uris.DCIM_iDRACCardService,
'iDRACReset',
selectors,
properties,
check_return_value=False)
message_id = utils.find_xml(doc,
'MessageID',
uris.DCIM_iDRACCardService).text
return "RAC064" == message_id
def _name_formatter(attribute):
return "{}#{}".format(attribute.group_id, attribute.name)

View File

@ -55,6 +55,9 @@ DCIM_iDRACCardService = ('http://schemas.dell.com/wbem/wscim/1/cim-schema/2/'
DCIM_iDRACCardString = ('http://schemas.dell.com/wbem/wscim/1/cim-schema/2/'
'DCIM_iDRACCardString')
DCIM_JobService = ('http://schemas.dell.com/wbem/wscim/1/cim-schema/2/'
'DCIM_JobService')
DCIM_LCEnumeration = ('http://schemas.dell.com/wbem/wscim/1/cim-schema/2/'
'DCIM_LCEnumeration')

View File

@ -312,3 +312,292 @@ class ClientiDRACCardChangesTestCase(base.BaseTest):
cim_creation_class_name='DCIM_iDRACCardService',
cim_name='DCIM:iDRACCardService',
target=dracclient.client.DRACClient.IDRAC_FQDD)
class ClientiDRACCardResetTestCase(base.BaseTest):
def setUp(self):
super(ClientiDRACCardResetTestCase, self).setUp()
self.drac_client = dracclient.client.DRACClient(
**test_utils.FAKE_ENDPOINT)
@mock.patch('dracclient.client.subprocess.call')
def test_ping_host(self, mock_os_system):
mock_os_system.return_value = 0
response = self.drac_client._ping_host('127.0.0.1')
self.assertEqual(mock_os_system.call_count, 1)
self.assertEqual(True, response)
@mock.patch('dracclient.client.subprocess.call')
def test_ping_host_not_pingable(self, mock_os_system):
mock_os_system.return_value = 1
response = self.drac_client._ping_host('127.0.0.1')
self.assertEqual(mock_os_system.call_count, 1)
self.assertEqual(False, response)
@mock.patch('dracclient.client.subprocess.call')
def test_ping_host_name_not_known(self, mock_os_system):
mock_os_system.return_value = 2
response = self.drac_client._ping_host('127.0.0.1')
self.assertEqual(mock_os_system.call_count, 1)
self.assertEqual(False, response)
@mock.patch('time.sleep')
@mock.patch('dracclient.client.DRACClient._ping_host')
def test_wait_for_host_alive(self, mock_ping_host, mock_sleep):
total_calls = 5
ping_count = 3
mock_ping_host.return_value = True
mock_sleep.return_value = None
response = self.drac_client._wait_for_host_state(
'hostname',
alive=True,
ping_count=ping_count,
retries=total_calls)
self.assertEqual(True, response)
self.assertEqual(mock_sleep.call_count, ping_count)
self.assertEqual(mock_ping_host.call_count, ping_count)
@mock.patch('time.sleep')
@mock.patch('dracclient.client.DRACClient._ping_host')
def test_wait_for_host_alive_fail(self, mock_ping_host, mock_sleep):
total_calls = 5
ping_count = 3
mock_ping_host.return_value = False
mock_sleep.return_value = None
response = self.drac_client._wait_for_host_state(
'hostname',
alive=True,
ping_count=ping_count,
retries=total_calls)
self.assertEqual(False, response)
self.assertEqual(mock_sleep.call_count, total_calls)
self.assertEqual(mock_ping_host.call_count, total_calls)
@mock.patch('time.sleep')
@mock.patch('dracclient.client.DRACClient._ping_host')
def test_wait_for_host_dead(self, mock_ping_host, mock_sleep):
total_calls = 5
ping_count = 3
mock_ping_host.return_value = False
mock_sleep.return_value = None
response = self.drac_client._wait_for_host_state(
'hostname',
alive=False,
ping_count=ping_count,
retries=total_calls)
self.assertEqual(True, response)
self.assertEqual(mock_sleep.call_count, ping_count)
self.assertEqual(mock_ping_host.call_count, ping_count)
@mock.patch('time.sleep')
@mock.patch('dracclient.client.DRACClient._ping_host')
def test_wait_for_host_dead_fail(self, mock_ping_host, mock_sleep):
total_calls = 5
ping_count = 3
mock_ping_host.return_value = True
mock_sleep.return_value = None
response = self.drac_client._wait_for_host_state(
'hostname',
alive=False,
ping_count=ping_count,
retries=total_calls)
self.assertEqual(False, response)
self.assertEqual(mock_sleep.call_count, total_calls)
self.assertEqual(mock_ping_host.call_count, total_calls)
@mock.patch('time.sleep')
@mock.patch('dracclient.client.DRACClient._ping_host')
def test_wait_for_host_alive_with_intermittent(
self, mock_ping_host, mock_sleep):
total_calls = 6
ping_count = 3
mock_ping_host.side_effect = [True, True, False, True, True, True]
mock_sleep.return_value = None
response = self.drac_client._wait_for_host_state(
'hostname',
alive=True,
ping_count=ping_count,
retries=total_calls)
self.assertEqual(True, response)
self.assertEqual(mock_sleep.call_count, total_calls)
@mock.patch('time.sleep')
@mock.patch('dracclient.client.DRACClient._ping_host')
def test_wait_for_host_dead_with_intermittent(
self, mock_ping_host, mock_sleep):
total_calls = 6
ping_count = 3
mock_ping_host.side_effect = [False, False, True, False, False, False]
mock_sleep.return_value = None
response = self.drac_client._wait_for_host_state(
'hostname',
alive=False,
ping_count=ping_count,
retries=total_calls)
self.assertEqual(True, response)
self.assertEqual(mock_sleep.call_count, total_calls)
@mock.patch.object(dracclient.client.WSManClient, 'invoke', spec_set=True,
autospec=True)
def test_reset_idrac(self, mock_invoke):
expected_selectors = {
'CreationClassName': "DCIM_iDRACCardService",
'Name': "DCIM:iDRACCardService",
'SystemCreationClassName': 'DCIM_ComputerSystem',
'SystemName': 'DCIM:ComputerSystem'}
expected_properties = {'Force': '0'}
mock_invoke.return_value = lxml.etree.fromstring(
test_utils.iDracCardInvocations[uris.DCIM_iDRACCardService][
'iDRACReset']['ok'])
result = self.drac_client.reset_idrac()
mock_invoke.assert_called_once_with(
mock.ANY, uris.DCIM_iDRACCardService, 'iDRACReset',
expected_selectors, expected_properties,
check_return_value=False)
self.assertTrue(result)
@mock.patch.object(dracclient.client.WSManClient, 'invoke', spec_set=True,
autospec=True)
def test_reset_idrac_force(self, mock_invoke):
expected_selectors = {
'CreationClassName': "DCIM_iDRACCardService",
'Name': "DCIM:iDRACCardService",
'SystemCreationClassName': 'DCIM_ComputerSystem',
'SystemName': 'DCIM:ComputerSystem'}
expected_properties = {'Force': '1'}
mock_invoke.return_value = lxml.etree.fromstring(
test_utils.iDracCardInvocations[uris.DCIM_iDRACCardService][
'iDRACReset']['ok'])
result = self.drac_client.reset_idrac(force=True)
mock_invoke.assert_called_once_with(
mock.ANY, uris.DCIM_iDRACCardService, 'iDRACReset',
expected_selectors, expected_properties,
check_return_value=False)
self.assertTrue(result)
@mock.patch.object(dracclient.client.WSManClient, 'invoke', spec_set=True,
autospec=True)
def test_reset_idrac_bad_result(self, mock_invoke):
expected_selectors = {
'CreationClassName': "DCIM_iDRACCardService",
'Name': "DCIM:iDRACCardService",
'SystemCreationClassName': 'DCIM_ComputerSystem',
'SystemName': 'DCIM:ComputerSystem'}
expected_properties = {'Force': '0'}
expected_message = ("Failed to reset iDRAC")
mock_invoke.return_value = lxml.etree.fromstring(
test_utils.iDracCardInvocations[uris.DCIM_iDRACCardService][
'iDRACReset']['error'])
self.assertRaisesRegexp(
exceptions.DRACOperationFailed, re.escape(expected_message),
self.drac_client.reset_idrac)
mock_invoke.assert_called_once_with(
mock.ANY, uris.DCIM_iDRACCardService, 'iDRACReset',
expected_selectors, expected_properties,
check_return_value=False)
@mock.patch('time.sleep')
@mock.patch('dracclient.client.WSManClient.wait_until_idrac_is_ready')
@mock.patch('dracclient.client.DRACClient._wait_for_host_state')
@mock.patch(
'dracclient.client.idrac_card.iDRACCardConfiguration.reset_idrac')
def test_reset_idrac_wait(
self,
mock_reset_idrac,
mock_wait_for_host_state,
mock_wait_until_idrac_is_ready,
mock_sleep):
mock_reset_idrac.return_value = True
mock_wait_for_host_state.side_effect = [True, True]
mock_wait_until_idrac_is_ready.return_value = True
mock_sleep.return_value = None
self.drac_client.reset_idrac(wait=True)
mock_reset_idrac.assert_called_once()
self.assertEqual(mock_wait_for_host_state.call_count, 2)
mock_wait_until_idrac_is_ready.assert_called_once()
@mock.patch('time.sleep')
@mock.patch('dracclient.client.WSManClient.wait_until_idrac_is_ready')
@mock.patch('dracclient.client.DRACClient._wait_for_host_state')
@mock.patch(
'dracclient.client.idrac_card.iDRACCardConfiguration.reset_idrac')
def test_reset_idrac_wait_failed_reset(
self,
mock_reset_idrac,
mock_wait_for_host_state,
mock_wait_until_idrac_is_ready,
mock_sleep):
mock_reset_idrac.return_value = False
mock_wait_for_host_state.side_effect = [True, True]
mock_wait_until_idrac_is_ready.return_value = False
mock_sleep.return_value = None
expected_message = ("Failed to reset iDRAC")
self.assertRaisesRegexp(
exceptions.DRACOperationFailed, re.escape(expected_message),
self.drac_client.reset_idrac, wait=True)
mock_reset_idrac.assert_called_once()
mock_wait_for_host_state.assert_not_called()
mock_wait_until_idrac_is_ready.assert_not_called()
@mock.patch('time.sleep')
@mock.patch('dracclient.client.WSManClient.wait_until_idrac_is_ready')
@mock.patch('dracclient.client.DRACClient._wait_for_host_state')
@mock.patch(
'dracclient.client.idrac_card.iDRACCardConfiguration.reset_idrac')
def test_reset_idrac_fail_wait_not_pingable(
self,
mock_reset_idrac,
mock_wait_for_host_state,
mock_wait_until_idrac_is_ready,
mock_sleep):
mock_reset_idrac.return_value = True
mock_wait_for_host_state.side_effect = [False, True]
mock_wait_until_idrac_is_ready.return_value = True
mock_sleep.return_value = None
expected_message = (
"Timed out waiting for the 1.2.3.4 iDRAC to become not pingable")
self.assertRaisesRegexp(
exceptions.DRACOperationFailed, re.escape(expected_message),
self.drac_client.reset_idrac, wait=True)
mock_reset_idrac.assert_called_once()
mock_wait_for_host_state.assert_called_once()
mock_wait_until_idrac_is_ready.assert_not_called()
@mock.patch('time.sleep')
@mock.patch('dracclient.client.WSManClient.wait_until_idrac_is_ready')
@mock.patch('dracclient.client.DRACClient._wait_for_host_state')
@mock.patch(
'dracclient.client.idrac_card.iDRACCardConfiguration.reset_idrac')
def test_reset_idrac_fail_wait_pingable(
self,
mock_reset_idrac,
mock_wait_for_host_state,
mock_wait_until_idrac_is_ready,
mock_sleep):
mock_reset_idrac.return_value = True
mock_wait_for_host_state.side_effect = [True, False]
mock_wait_until_idrac_is_ready.return_value = True
mock_sleep.return_value = None
expected_message = (
"Timed out waiting for the 1.2.3.4 iDRAC to become pingable")
self.assertRaisesRegexp(
exceptions.DRACOperationFailed, re.escape(expected_message),
self.drac_client.reset_idrac, wait=True)
mock_reset_idrac.assert_called_once()
self.assertEqual(mock_wait_for_host_state.call_count, 2)
mock_wait_until_idrac_is_ready.assert_not_called()

View File

@ -34,6 +34,7 @@ def load_wsman_xml(name):
return xml_body
WSManEnumerations = {
'context': [
load_wsman_xml('wsman-enum_context-1'),
@ -152,7 +153,14 @@ iDracCardInvocations = {
'SetAttributes': {
'ok': load_wsman_xml(
'idrac_service-invoke-set_attributes-ok')
},
'iDRACReset': {
'ok': load_wsman_xml(
'idrac_service-reset-ok'),
'error': load_wsman_xml(
'idrac_service-reset-error')
}
}
}

View File

@ -0,0 +1,22 @@
<s:Envelope xmlns:s="http://www.w3.org/2003/05/soap-envelope"
xmlns:wsa="http://schemas.xmlsoap.org/ws/2004/08/addressing"
xmlns:n1="http://schemas.dell.com/wbem/wscim/1/cim-schema/2/DCIM_iDRACCardService">
<s:Header>
<wsa:To>http://schemas.xmlsoap.org/ws/2004/08/addressing/role/anonymous
</wsa:To>
<wsa:Action>http://schemas.dell.com/wbem/wscim/1/cim-schema/2/DCIM_iDRACCardService/iDRACResetResponse
</wsa:Action>
<wsa:RelatesTo>uuid:a65ce3df-3690-42dd-af45-5c1f2cd0793b
</wsa:RelatesTo>
<wsa:MessageID>uuid:e8f2cbe0-6fd0-1fd0-8057-dc9c046694d0
</wsa:MessageID>
</s:Header>
<s:Body>
<n1:iDRACReset_OUTPUT>
<n1:Message>Invalid parameter value for Force</n1:Message>
<n1:MessageArguments>Force</n1:MessageArguments>
<n1:MessageID>RAC004</n1:MessageID>
<n1:ReturnValue>2</n1:ReturnValue>
</n1:iDRACReset_OUTPUT>
</s:Body>
</s:Envelope>

View File

@ -0,0 +1,22 @@
<s:Envelope xmlns:s="http://www.w3.org/2003/05/soap-envelope"
xmlns:wsa="http://schemas.xmlsoap.org/ws/2004/08/addressing"
xmlns:n1="http://schemas.dell.com/wbem/wscim/1/cim-schema/2/DCIM_iDRACCardService">
<s:Header>
<wsa:To>http://schemas.xmlsoap.org/ws/2004/08/addressing/role/anonymous
</wsa:To>
<wsa:Action>http://schemas.dell.com/wbem/wscim/1/cim-schema/2/DCIM_iDRACCardService/iDRACResetResponse
</wsa:Action>
<wsa:RelatesTo>uuid:a4a1cd1a-7c10-4dfc-98d9-d0cc2cd7c80c
</wsa:RelatesTo>
<wsa:MessageID>uuid:6f9ecf40-6fd1-1fd1-a60b-dc9c046694d0
</wsa:MessageID>
</s:Header>
<s:Body>
<n1:iDRACReset_OUTPUT>
<n1:Message>iDRAC was successfully reset.</n1:Message>
<n1:MessageID>RAC064</n1:MessageID>
<n1:ReturnValue>0</n1:ReturnValue>
</n1:iDRACReset_OUTPUT>
</s:Body>
</s:Envelope>