Adds ability to reset iDRAC

Adds new function to reset the iDRAC and wait for it
to become operational again.

Change-Id: Ia8dc0b97e02fc5f2c4d39b6b6d90456c1cfc5b7a
Co-Authored-By: Christopher Dearborn <christopher.dearborn@dell.com>

dracclient/client.py

@@ -16,6 +16,7 @@ Wrapper for pywsman.Client
 """
 
 import logging
+import subprocess
 import time
 
 from dracclient import constants
@@ -241,6 +242,103 @@ class DRACClient(object):
         """
         return self._idrac_cfg.set_idrac_settings(settings, idrac_fqdd)
 
+    def reset_idrac(self, force=False, wait=False,
+                    ready_wait_time=30):
+        """Resets the iDRAC and optionally block until reset is complete.
+
+        :param force: does a force reset when True and a graceful reset when
+               False
+        :param wait: returns immediately after reset if False, or waits
+                for the iDRAC to return to operational state if True
+        :param ready_wait_time: the amount of time in seconds to wait after
+                the reset before starting to check on the iDRAC's status
+        :returns: True on success, raises exception on failure
+        :raises: WSManRequestFailure on request failures
+        :raises: WSManInvalidResponse when receiving invalid response
+        :raises: DRACOperationFailed on failure to reset iDRAC
+        """
+        return_value = self._idrac_cfg.reset_idrac(force)
+        if not wait and return_value:
+            return return_value
+
+        if not return_value:
+            raise exceptions.DRACOperationFailed(
+                drac_messages="Failed to reset iDRAC")
+
+        LOG.debug("iDRAC was reset, waiting for return to operational state")
+
+        state_reached = self._wait_for_host_state(
+            self.client.host,
+            alive=False,
+            ping_count=3,
+            retries=24)
+
+        if not state_reached:
+            raise exceptions.DRACOperationFailed(
+                drac_messages="Timed out waiting for the %s iDRAC to become "
+                "not pingable" % self.client.host)
+
+        LOG.info("The iDRAC has become not pingable")
+
+        state_reached = self._wait_for_host_state(
+            self.client.host,
+            alive=True,
+            ping_count=3,
+            retries=24)
+
+        if not state_reached:
+            raise exceptions.DRACOperationFailed(
+                drac_messages="Timed out waiting for the %s iDRAC to become "
+                "pingable" % self.client.host)
+
+        LOG.info("The iDRAC has become pingable")
+        LOG.info("Waiting for the iDRAC to become ready")
+        time.sleep(ready_wait_time)
+
+        self.client.wait_until_idrac_is_ready()
+
+    def _ping_host(self, host):
+        response = subprocess.call(
+            "ping -c 1 {} 2>&1 1>/dev/null".format(host), shell=True)
+        return (response == 0)
+
+    def _wait_for_host_state(self,
+                             host,
+                             alive=True,
+                             ping_count=3,
+                             retries=24):
+        if alive:
+            ping_type = "pingable"
+
+        else:
+            ping_type = "not pingable"
+
+        LOG.info("Waiting for the iDRAC to become %s", ping_type)
+
+        response_count = 0
+        state_reached = False
+
+        while retries > 0 and not state_reached:
+            response = self._ping_host(host)
+            retries -= 1
+            if response == alive:
+                response_count += 1
+                LOG.debug("The iDRAC is %s, count=%s",
+                          ping_type,
+                          response_count)
+                if response_count == ping_count:
+                    LOG.debug("Reached specified ping count")
+                    state_reached = True
+            else:
+                response_count = 0
+                if alive:
+                    LOG.debug("The iDRAC is still not pingable")
+                else:
+                    LOG.debug("The iDRAC is still pingable")
+            time.sleep(10)
+
+        return state_reached
+
     def commit_pending_idrac_changes(
             self,
             idrac_fqdd=IDRAC_FQDD,

dracclient/resources/idrac_card.py

@@ -321,6 +321,33 @@ class iDRACCardConfiguration(object):
                                   idrac_fqdd,
                                   name_formatter=_name_formatter)
 
+    def reset_idrac(self, force=False):
+        """Resets the iDRAC
+
+        :param force: does a force reset when True and a graceful reset when
+               False.
+        :returns: True on success and False on failure.
+        :raises: WSManRequestFailure on request failures
+        :raises: WSManInvalidResponse when receiving invalid response
+        """
+        selectors = {'CreationClassName': "DCIM_iDRACCardService",
+                     'Name': "DCIM:iDRACCardService",
+                     'SystemCreationClassName': 'DCIM_ComputerSystem',
+                     'SystemName': 'DCIM:ComputerSystem'}
+
+        properties = {'Force': "1" if force else "0"}
+
+        doc = self.client.invoke(uris.DCIM_iDRACCardService,
+                                 'iDRACReset',
+                                 selectors,
+                                 properties,
+                                 check_return_value=False)
+
+        message_id = utils.find_xml(doc,
+                                    'MessageID',
+                                    uris.DCIM_iDRACCardService).text
+        return "RAC064" == message_id
+
 
 def _name_formatter(attribute):
     return "{}#{}".format(attribute.group_id, attribute.name)

dracclient/resources/uris.py

@@ -55,6 +55,9 @@ DCIM_iDRACCardService = ('http://schemas.dell.com/wbem/wscim/1/cim-schema/2/'
 DCIM_iDRACCardString = ('http://schemas.dell.com/wbem/wscim/1/cim-schema/2/'
                         'DCIM_iDRACCardString')
 
+DCIM_JobService = ('http://schemas.dell.com/wbem/wscim/1/cim-schema/2/'
+                   'DCIM_JobService')
+
 DCIM_LCEnumeration = ('http://schemas.dell.com/wbem/wscim/1/cim-schema/2/'
                       'DCIM_LCEnumeration')
 

dracclient/tests/test_idrac_card.py

@@ -312,3 +312,292 @@ class ClientiDRACCardChangesTestCase(base.BaseTest):
             cim_creation_class_name='DCIM_iDRACCardService',
             cim_name='DCIM:iDRACCardService',
             target=dracclient.client.DRACClient.IDRAC_FQDD)
+
+
+class ClientiDRACCardResetTestCase(base.BaseTest):
+
+    def setUp(self):
+        super(ClientiDRACCardResetTestCase, self).setUp()
+        self.drac_client = dracclient.client.DRACClient(
+            **test_utils.FAKE_ENDPOINT)
+
+    @mock.patch('dracclient.client.subprocess.call')
+    def test_ping_host(self, mock_os_system):
+        mock_os_system.return_value = 0
+        response = self.drac_client._ping_host('127.0.0.1')
+        self.assertEqual(mock_os_system.call_count, 1)
+        self.assertEqual(True, response)
+
+    @mock.patch('dracclient.client.subprocess.call')
+    def test_ping_host_not_pingable(self, mock_os_system):
+        mock_os_system.return_value = 1
+        response = self.drac_client._ping_host('127.0.0.1')
+        self.assertEqual(mock_os_system.call_count, 1)
+        self.assertEqual(False, response)
+
+    @mock.patch('dracclient.client.subprocess.call')
+    def test_ping_host_name_not_known(self, mock_os_system):
+        mock_os_system.return_value = 2
+        response = self.drac_client._ping_host('127.0.0.1')
+        self.assertEqual(mock_os_system.call_count, 1)
+        self.assertEqual(False, response)
+
+    @mock.patch('time.sleep')
+    @mock.patch('dracclient.client.DRACClient._ping_host')
+    def test_wait_for_host_alive(self, mock_ping_host, mock_sleep):
+        total_calls = 5
+        ping_count = 3
+        mock_ping_host.return_value = True
+        mock_sleep.return_value = None
+        response = self.drac_client._wait_for_host_state(
+            'hostname',
+            alive=True,
+            ping_count=ping_count,
+            retries=total_calls)
+        self.assertEqual(True, response)
+        self.assertEqual(mock_sleep.call_count, ping_count)
+        self.assertEqual(mock_ping_host.call_count, ping_count)
+
+    @mock.patch('time.sleep')
+    @mock.patch('dracclient.client.DRACClient._ping_host')
+    def test_wait_for_host_alive_fail(self, mock_ping_host, mock_sleep):
+        total_calls = 5
+        ping_count = 3
+        mock_ping_host.return_value = False
+        mock_sleep.return_value = None
+        response = self.drac_client._wait_for_host_state(
+            'hostname',
+            alive=True,
+            ping_count=ping_count,
+            retries=total_calls)
+        self.assertEqual(False, response)
+        self.assertEqual(mock_sleep.call_count, total_calls)
+        self.assertEqual(mock_ping_host.call_count, total_calls)
+
+    @mock.patch('time.sleep')
+    @mock.patch('dracclient.client.DRACClient._ping_host')
+    def test_wait_for_host_dead(self, mock_ping_host, mock_sleep):
+        total_calls = 5
+        ping_count = 3
+        mock_ping_host.return_value = False
+        mock_sleep.return_value = None
+        response = self.drac_client._wait_for_host_state(
+            'hostname',
+            alive=False,
+            ping_count=ping_count,
+            retries=total_calls)
+        self.assertEqual(True, response)
+        self.assertEqual(mock_sleep.call_count, ping_count)
+        self.assertEqual(mock_ping_host.call_count, ping_count)
+
+    @mock.patch('time.sleep')
+    @mock.patch('dracclient.client.DRACClient._ping_host')
+    def test_wait_for_host_dead_fail(self, mock_ping_host, mock_sleep):
+        total_calls = 5
+        ping_count = 3
+        mock_ping_host.return_value = True
+        mock_sleep.return_value = None
+        response = self.drac_client._wait_for_host_state(
+            'hostname',
+            alive=False,
+            ping_count=ping_count,
+            retries=total_calls)
+        self.assertEqual(False, response)
+        self.assertEqual(mock_sleep.call_count, total_calls)
+        self.assertEqual(mock_ping_host.call_count, total_calls)
+
+    @mock.patch('time.sleep')
+    @mock.patch('dracclient.client.DRACClient._ping_host')
+    def test_wait_for_host_alive_with_intermittent(
+            self, mock_ping_host, mock_sleep):
+        total_calls = 6
+        ping_count = 3
+        mock_ping_host.side_effect = [True, True, False, True, True, True]
+        mock_sleep.return_value = None
+        response = self.drac_client._wait_for_host_state(
+            'hostname',
+            alive=True,
+            ping_count=ping_count,
+            retries=total_calls)
+        self.assertEqual(True, response)
+        self.assertEqual(mock_sleep.call_count, total_calls)
+
+    @mock.patch('time.sleep')
+    @mock.patch('dracclient.client.DRACClient._ping_host')
+    def test_wait_for_host_dead_with_intermittent(
+            self, mock_ping_host, mock_sleep):
+        total_calls = 6
+        ping_count = 3
+        mock_ping_host.side_effect = [False, False, True, False, False, False]
+        mock_sleep.return_value = None
+        response = self.drac_client._wait_for_host_state(
+            'hostname',
+            alive=False,
+            ping_count=ping_count,
+            retries=total_calls)
+        self.assertEqual(True, response)
+        self.assertEqual(mock_sleep.call_count, total_calls)
+
+    @mock.patch.object(dracclient.client.WSManClient, 'invoke', spec_set=True,
+                       autospec=True)
+    def test_reset_idrac(self, mock_invoke):
+        expected_selectors = {
+            'CreationClassName': "DCIM_iDRACCardService",
+            'Name': "DCIM:iDRACCardService",
+            'SystemCreationClassName': 'DCIM_ComputerSystem',
+            'SystemName': 'DCIM:ComputerSystem'}
+        expected_properties = {'Force': '0'}
+        mock_invoke.return_value = lxml.etree.fromstring(
+            test_utils.iDracCardInvocations[uris.DCIM_iDRACCardService][
+                'iDRACReset']['ok'])
+
+        result = self.drac_client.reset_idrac()
+
+        mock_invoke.assert_called_once_with(
+            mock.ANY, uris.DCIM_iDRACCardService, 'iDRACReset',
+            expected_selectors, expected_properties,
+            check_return_value=False)
+        self.assertTrue(result)
+
+    @mock.patch.object(dracclient.client.WSManClient, 'invoke', spec_set=True,
+                       autospec=True)
+    def test_reset_idrac_force(self, mock_invoke):
+        expected_selectors = {
+            'CreationClassName': "DCIM_iDRACCardService",
+            'Name': "DCIM:iDRACCardService",
+            'SystemCreationClassName': 'DCIM_ComputerSystem',
+            'SystemName': 'DCIM:ComputerSystem'}
+        expected_properties = {'Force': '1'}
+        mock_invoke.return_value = lxml.etree.fromstring(
+            test_utils.iDracCardInvocations[uris.DCIM_iDRACCardService][
+                'iDRACReset']['ok'])
+
+        result = self.drac_client.reset_idrac(force=True)
+
+        mock_invoke.assert_called_once_with(
+            mock.ANY, uris.DCIM_iDRACCardService, 'iDRACReset',
+            expected_selectors, expected_properties,
+            check_return_value=False)
+        self.assertTrue(result)
+
+    @mock.patch.object(dracclient.client.WSManClient, 'invoke', spec_set=True,
+                       autospec=True)
+    def test_reset_idrac_bad_result(self, mock_invoke):
+        expected_selectors = {
+            'CreationClassName': "DCIM_iDRACCardService",
+            'Name': "DCIM:iDRACCardService",
+            'SystemCreationClassName': 'DCIM_ComputerSystem',
+            'SystemName': 'DCIM:ComputerSystem'}
+        expected_properties = {'Force': '0'}
+        expected_message = ("Failed to reset iDRAC")
+        mock_invoke.return_value = lxml.etree.fromstring(
+            test_utils.iDracCardInvocations[uris.DCIM_iDRACCardService][
+                'iDRACReset']['error'])
+
+        self.assertRaisesRegexp(
+            exceptions.DRACOperationFailed, re.escape(expected_message),
+            self.drac_client.reset_idrac)
+
+        mock_invoke.assert_called_once_with(
+            mock.ANY, uris.DCIM_iDRACCardService, 'iDRACReset',
+            expected_selectors, expected_properties,
+            check_return_value=False)
+
+    @mock.patch('time.sleep')
+    @mock.patch('dracclient.client.WSManClient.wait_until_idrac_is_ready')
+    @mock.patch('dracclient.client.DRACClient._wait_for_host_state')
+    @mock.patch(
+        'dracclient.client.idrac_card.iDRACCardConfiguration.reset_idrac')
+    def test_reset_idrac_wait(
+            self,
+            mock_reset_idrac,
+            mock_wait_for_host_state,
+            mock_wait_until_idrac_is_ready,
+            mock_sleep):
+        mock_reset_idrac.return_value = True
+        mock_wait_for_host_state.side_effect = [True, True]
+        mock_wait_until_idrac_is_ready.return_value = True
+        mock_sleep.return_value = None
+
+        self.drac_client.reset_idrac(wait=True)
+
+        mock_reset_idrac.assert_called_once()
+        self.assertEqual(mock_wait_for_host_state.call_count, 2)
+        mock_wait_until_idrac_is_ready.assert_called_once()
+
+    @mock.patch('time.sleep')
+    @mock.patch('dracclient.client.WSManClient.wait_until_idrac_is_ready')
+    @mock.patch('dracclient.client.DRACClient._wait_for_host_state')
+    @mock.patch(
+        'dracclient.client.idrac_card.iDRACCardConfiguration.reset_idrac')
+    def test_reset_idrac_wait_failed_reset(
+            self,
+            mock_reset_idrac,
+            mock_wait_for_host_state,
+            mock_wait_until_idrac_is_ready,
+            mock_sleep):
+        mock_reset_idrac.return_value = False
+        mock_wait_for_host_state.side_effect = [True, True]
+        mock_wait_until_idrac_is_ready.return_value = False
+        mock_sleep.return_value = None
+        expected_message = ("Failed to reset iDRAC")
+
+        self.assertRaisesRegexp(
+            exceptions.DRACOperationFailed, re.escape(expected_message),
+            self.drac_client.reset_idrac, wait=True)
+
+        mock_reset_idrac.assert_called_once()
+        mock_wait_for_host_state.assert_not_called()
+        mock_wait_until_idrac_is_ready.assert_not_called()
+
+    @mock.patch('time.sleep')
+    @mock.patch('dracclient.client.WSManClient.wait_until_idrac_is_ready')
+    @mock.patch('dracclient.client.DRACClient._wait_for_host_state')
+    @mock.patch(
+        'dracclient.client.idrac_card.iDRACCardConfiguration.reset_idrac')
+    def test_reset_idrac_fail_wait_not_pingable(
+            self,
+            mock_reset_idrac,
+            mock_wait_for_host_state,
+            mock_wait_until_idrac_is_ready,
+            mock_sleep):
+        mock_reset_idrac.return_value = True
+        mock_wait_for_host_state.side_effect = [False, True]
+        mock_wait_until_idrac_is_ready.return_value = True
+        mock_sleep.return_value = None
+        expected_message = (
+            "Timed out waiting for the 1.2.3.4 iDRAC to become not pingable")
+
+        self.assertRaisesRegexp(
+            exceptions.DRACOperationFailed, re.escape(expected_message),
+            self.drac_client.reset_idrac, wait=True)
+
+        mock_reset_idrac.assert_called_once()
+        mock_wait_for_host_state.assert_called_once()
+        mock_wait_until_idrac_is_ready.assert_not_called()
+
+    @mock.patch('time.sleep')
+    @mock.patch('dracclient.client.WSManClient.wait_until_idrac_is_ready')
+    @mock.patch('dracclient.client.DRACClient._wait_for_host_state')
+    @mock.patch(
+        'dracclient.client.idrac_card.iDRACCardConfiguration.reset_idrac')
+    def test_reset_idrac_fail_wait_pingable(
+            self,
+            mock_reset_idrac,
+            mock_wait_for_host_state,
+            mock_wait_until_idrac_is_ready,
+            mock_sleep):
+        mock_reset_idrac.return_value = True
+        mock_wait_for_host_state.side_effect = [True, False]
+        mock_wait_until_idrac_is_ready.return_value = True
+        mock_sleep.return_value = None
+        expected_message = (
+            "Timed out waiting for the 1.2.3.4 iDRAC to become pingable")
+
+        self.assertRaisesRegexp(
+            exceptions.DRACOperationFailed, re.escape(expected_message),
+            self.drac_client.reset_idrac, wait=True)
+
+        mock_reset_idrac.assert_called_once()
+        self.assertEqual(mock_wait_for_host_state.call_count, 2)
+        mock_wait_until_idrac_is_ready.assert_not_called()

dracclient/tests/utils.py

@@ -34,6 +34,7 @@ def load_wsman_xml(name):
 
     return xml_body
 
+
 WSManEnumerations = {
     'context': [
         load_wsman_xml('wsman-enum_context-1'),
@@ -152,7 +153,14 @@ iDracCardInvocations = {
         'SetAttributes': {
             'ok': load_wsman_xml(
                 'idrac_service-invoke-set_attributes-ok')
+        },
+        'iDRACReset': {
+            'ok': load_wsman_xml(
+                'idrac_service-reset-ok'),
+            'error': load_wsman_xml(
+                'idrac_service-reset-error')
         }
+
     }
 }
 

dracclient/tests/wsman_mocks/idrac_service-reset-error.xml

@@ -0,0 +1,22 @@
+<s:Envelope xmlns:s="http://www.w3.org/2003/05/soap-envelope"
+	xmlns:wsa="http://schemas.xmlsoap.org/ws/2004/08/addressing"
+	xmlns:n1="http://schemas.dell.com/wbem/wscim/1/cim-schema/2/DCIM_iDRACCardService">
+	<s:Header>
+		<wsa:To>http://schemas.xmlsoap.org/ws/2004/08/addressing/role/anonymous
+		</wsa:To>
+		<wsa:Action>http://schemas.dell.com/wbem/wscim/1/cim-schema/2/DCIM_iDRACCardService/iDRACResetResponse
+		</wsa:Action>
+		<wsa:RelatesTo>uuid:a65ce3df-3690-42dd-af45-5c1f2cd0793b
+		</wsa:RelatesTo>
+		<wsa:MessageID>uuid:e8f2cbe0-6fd0-1fd0-8057-dc9c046694d0
+		</wsa:MessageID>
+	</s:Header>
+	<s:Body>
+		<n1:iDRACReset_OUTPUT>
+			<n1:Message>Invalid parameter value for Force</n1:Message>
+			<n1:MessageArguments>Force</n1:MessageArguments>
+			<n1:MessageID>RAC004</n1:MessageID>
+			<n1:ReturnValue>2</n1:ReturnValue>
+		</n1:iDRACReset_OUTPUT>
+	</s:Body>
+</s:Envelope>

dracclient/tests/wsman_mocks/idrac_service-reset-ok.xml

@@ -0,0 +1,22 @@
+<s:Envelope xmlns:s="http://www.w3.org/2003/05/soap-envelope"
+	xmlns:wsa="http://schemas.xmlsoap.org/ws/2004/08/addressing"
+	xmlns:n1="http://schemas.dell.com/wbem/wscim/1/cim-schema/2/DCIM_iDRACCardService">
+	<s:Header>
+		<wsa:To>http://schemas.xmlsoap.org/ws/2004/08/addressing/role/anonymous
+		</wsa:To>
+		<wsa:Action>http://schemas.dell.com/wbem/wscim/1/cim-schema/2/DCIM_iDRACCardService/iDRACResetResponse
+		</wsa:Action>
+		<wsa:RelatesTo>uuid:a4a1cd1a-7c10-4dfc-98d9-d0cc2cd7c80c
+		</wsa:RelatesTo>
+		<wsa:MessageID>uuid:6f9ecf40-6fd1-1fd1-a60b-dc9c046694d0
+		</wsa:MessageID>
+	</s:Header>
+	<s:Body>
+		<n1:iDRACReset_OUTPUT>
+			<n1:Message>iDRAC was successfully reset.</n1:Message>
+			<n1:MessageID>RAC064</n1:MessageID>
+			<n1:ReturnValue>0</n1:ReturnValue>
+		</n1:iDRACReset_OUTPUT>
+	</s:Body>
+</s:Envelope>
+