Invoke operations can wait until iDRAC is ready

Web Services Management (WS-Management and WS-Man) Invoke operations can
fail when issued to an integrated Dell Remote Access Controller (iDRAC)
whose Lifecycle Controller remote service is not "ready".

A Dell technical white paper [0], "Lifecycle Controller Integration --
Best Practices Guide", states that for Lifecycle Controller firmware
1.5.0 and later "The Lifecycle Controller remote service must be in a
'ready' state before running any other WSMAN commands." That applies to
almost all of the workflows and use cases documented by that paper and
supported by this project, openstack/python-dracclient. A notable
exception is the dracclient.client.WSManClient.is_idrac_ready() method,
which is a chicken and egg situation.

This patch adds a new parameter to the
dracclient.client.WSManClient.invoke() method that indicates whether or
not it should wait until the iDRAC is ready to accept commands before
issuing the Invoke command. When it is true, that method waits until the
iDRAC is ready before issuing the command. Since almost all Invoke
operations require the iDRAC to be ready, the new parameter's default
value is 'True'.

[0]
http://en.community.dell.com/techcenter/extras/m/white_papers/20442332

Change-Id: Ib5b9fb2a954579be40f47304c70157ab1f00d39c
Partial-Bug: #1697558
Related-Bug: #1691808

dracclient/client.py

@@ -567,7 +567,10 @@ class DRACClient(object):
 
 
 class WSManClient(wsman.Client):
-    """Wrapper for wsman.Client with return value checking"""
+    """Wrapper for wsman.Client that can wait until iDRAC is ready
+
+       Additionally, the Invoke operation offers return value checking.
+    """
 
     def __init__(
             self, host, username, password, port=443, path='/wsman',
@@ -601,7 +604,7 @@ class WSManClient(wsman.Client):
         self._ready_retry_delay = ready_retry_delay
 
     def invoke(self, resource_uri, method, selectors=None, properties=None,
-               expected_return_value=None):
+               expected_return_value=None, wait_for_idrac=True):
         """Invokes a remote WS-Man method
 
         :param resource_uri: URI of the resource
@@ -612,6 +615,9 @@ class WSManClient(wsman.Client):
             the DRAC card. For return value codes check the profile
             documentation of the resource used in the method call. If not set,
             return value checking is skipped.
+        :param wait_for_idrac: indicates whether or not to wait for the
+            iDRAC to be ready to accept commands before issuing the
+            command
         :returns: an lxml.etree.Element object of the response received
         :raises: WSManRequestFailure on request failures
         :raises: WSManInvalidResponse when receiving invalid response
@@ -619,6 +625,10 @@ class WSManClient(wsman.Client):
                  interface
         :raises: DRACUnexpectedReturnValue on return value mismatch
         """
+        if wait_for_idrac:
+            self.wait_until_idrac_is_ready(self._ready_retries,
+                                           self._ready_retry_delay)
+
         if selectors is None:
             selectors = {}
 
@@ -665,7 +675,8 @@ class WSManClient(wsman.Client):
                              'GetRemoteServicesAPIStatus',
                              selectors,
                              {},
-                             expected_return_value=utils.RET_SUCCESS)
+                             expected_return_value=utils.RET_SUCCESS,
+                             wait_for_idrac=False)
 
         message_id = utils.find_xml(result,
                                     'MessageID',

dracclient/tests/test_bios.py

@@ -43,7 +43,11 @@ class ClientPowerManagementTestCase(base.BaseTest):
 
         self.assertEqual('POWER_ON', self.drac_client.get_power_state())
 
-    def test_set_power_state(self, mock_requests):
+    @mock.patch.object(dracclient.client.WSManClient,
+                       'wait_until_idrac_is_ready', spec_set=True,
+                       autospec=True)
+    def test_set_power_state(self, mock_requests,
+                             mock_wait_until_idrac_is_ready):
         mock_requests.post(
             'https://1.2.3.4:443/wsman',
             text=test_utils.BIOSInvocations[
@@ -51,7 +55,11 @@ class ClientPowerManagementTestCase(base.BaseTest):
 
         self.assertIsNone(self.drac_client.set_power_state('POWER_ON'))
 
-    def test_set_power_state_fail(self, mock_requests):
+    @mock.patch.object(dracclient.client.WSManClient,
+                       'wait_until_idrac_is_ready', spec_set=True,
+                       autospec=True)
+    def test_set_power_state_fail(self, mock_requests,
+                                  mock_wait_until_idrac_is_ready):
         mock_requests.post(
             'https://1.2.3.4:443/wsman',
             text=test_utils.BIOSInvocations[
@@ -151,7 +159,11 @@ class ClientBootManagementTestCase(base.BaseTest):
             2,  boot_devices['IPL'][2].pending_assigned_sequence)
 
     @requests_mock.Mocker()
-    def test_change_boot_device_order(self, mock_requests):
+    @mock.patch.object(dracclient.client.WSManClient,
+                       'wait_until_idrac_is_ready', spec_set=True,
+                       autospec=True)
+    def test_change_boot_device_order(self, mock_requests,
+                                      mock_wait_until_idrac_is_ready):
         mock_requests.post(
             'https://1.2.3.4:443/wsman',
             text=test_utils.BIOSInvocations[
@@ -179,7 +191,11 @@ class ClientBootManagementTestCase(base.BaseTest):
             expected_properties, expected_return_value=utils.RET_SUCCESS)
 
     @requests_mock.Mocker()
-    def test_change_boot_device_order_error(self, mock_requests):
+    @mock.patch.object(dracclient.client.WSManClient,
+                       'wait_until_idrac_is_ready', spec_set=True,
+                       autospec=True)
+    def test_change_boot_device_order_error(self, mock_requests,
+                                            mock_wait_until_idrac_is_ready):
         mock_requests.post(
             'https://1.2.3.4:443/wsman',
             text=test_utils.BIOSInvocations[
@@ -341,7 +357,11 @@ class ClientBIOSConfigurationTestCase(base.BaseTest):
             expected_selectors, expected_properties)
 
     @requests_mock.Mocker()
-    def test_set_bios_settings_error(self, mock_requests):
+    @mock.patch.object(dracclient.client.WSManClient,
+                       'wait_until_idrac_is_ready', spec_set=True,
+                       autospec=True)
+    def test_set_bios_settings_error(self, mock_requests,
+                                     mock_wait_until_idrac_is_ready):
         mock_requests.post('https://1.2.3.4:443/wsman', [
             {'text': test_utils.BIOSEnumerations[
                 uris.DCIM_BIOSEnumeration]['ok']},

dracclient/tests/test_client.py

@@ -33,7 +33,10 @@ class WSManClientTestCase(base.BaseTest):
         resp = client.enumerate('http://resource')
         self.assertEqual('yay!', resp.text)
 
-    def test_invoke(self, mock_requests):
+    @mock.patch.object(dracclient.client.WSManClient,
+                       'wait_until_idrac_is_ready', spec_set=True,
+                       autospec=True)
+    def test_invoke(self, mock_requests, mock_wait_until_idrac_is_ready):
         xml = """
 <response xmlns:n1="http://resource">
     <n1:ReturnValue>42</n1:ReturnValue>
@@ -44,6 +47,27 @@ class WSManClientTestCase(base.BaseTest):
 
         client = dracclient.client.WSManClient(**test_utils.FAKE_ENDPOINT)
         resp = client.invoke('http://resource', 'Foo')
+        mock_wait_until_idrac_is_ready.assert_called_once_with(
+            client, constants.DEFAULT_IDRAC_IS_READY_RETRIES,
+            constants.DEFAULT_IDRAC_IS_READY_RETRY_DELAY_SEC)
+        self.assertEqual('yay!', resp.find('result').text)
+
+    @mock.patch.object(dracclient.client.WSManClient,
+                       'wait_until_idrac_is_ready', spec_set=True,
+                       autospec=True)
+    def test_invoke_without_wait_for_idrac(
+            self, mock_requests, mock_wait_until_idrac_is_ready):
+        xml = """
+<response xmlns:n1="http://resource">
+    <n1:ReturnValue>42</n1:ReturnValue>
+    <result>yay!</result>
+</response>
+"""  # noqa
+        mock_requests.post('https://1.2.3.4:443/wsman', text=xml)
+
+        client = dracclient.client.WSManClient(**test_utils.FAKE_ENDPOINT)
+        resp = client.invoke('http://resource', 'Foo', wait_for_idrac=False)
+        self.assertFalse(mock_wait_until_idrac_is_ready.called)
         self.assertEqual('yay!', resp.find('result').text)
 
     def test_invoke_with_expected_return_value(self, mock_requests):
@@ -57,7 +81,7 @@ class WSManClientTestCase(base.BaseTest):
 
         client = dracclient.client.WSManClient(**test_utils.FAKE_ENDPOINT)
         resp = client.invoke('http://resource', 'Foo',
-                             expected_return_value='42')
+                             expected_return_value='42', wait_for_idrac=False)
         self.assertEqual('yay!', resp.find('result').text)
 
     def test_invoke_with_error_return_value(self, mock_requests):
@@ -71,7 +95,7 @@ class WSManClientTestCase(base.BaseTest):
 
         client = dracclient.client.WSManClient(**test_utils.FAKE_ENDPOINT)
         self.assertRaises(exceptions.DRACOperationFailed, client.invoke,
-                          'http://resource', 'Foo')
+                          'http://resource', 'Foo', wait_for_idrac=False)
 
     def test_invoke_with_unexpected_return_value(self, mock_requests):
         xml = """
@@ -85,7 +109,7 @@ class WSManClientTestCase(base.BaseTest):
         client = dracclient.client.WSManClient(**test_utils.FAKE_ENDPOINT)
         self.assertRaises(exceptions.DRACUnexpectedReturnValue, client.invoke,
                           'http://resource', 'Foo',
-                          expected_return_value='4242')
+                          expected_return_value='4242', wait_for_idrac=False)
 
     def test_is_idrac_ready_ready(self, mock_requests):
         expected_text = test_utils.LifecycleControllerInvocations[

dracclient/tests/test_job.py

@@ -124,7 +124,11 @@ class ClientJobManagementTestCase(base.BaseTest):
         self.assertEqual('JID_442507917525', job_id)
 
     @requests_mock.Mocker()
-    def test_create_config_job_failed(self, mock_requests):
+    @mock.patch.object(dracclient.client.WSManClient,
+                       'wait_until_idrac_is_ready', spec_set=True,
+                       autospec=True)
+    def test_create_config_job_failed(self, mock_requests,
+                                      mock_wait_until_idrac_is_ready):
         cim_creation_class_name = 'DCIM_BIOSService'
         cim_name = 'DCIM:BIOSService'
         target = 'BIOS.Setup.1-1'
@@ -188,7 +192,11 @@ class ClientJobManagementTestCase(base.BaseTest):
             expected_return_value=utils.RET_SUCCESS)
 
     @requests_mock.Mocker()
-    def test_delete_pending_config_failed(self, mock_requests):
+    @mock.patch.object(dracclient.client.WSManClient,
+                       'wait_until_idrac_is_ready', spec_set=True,
+                       autospec=True)
+    def test_delete_pending_config_failed(self, mock_requests,
+                                          mock_wait_until_idrac_is_ready):
         cim_creation_class_name = 'DCIM_BIOSService'
         cim_name = 'DCIM:BIOSService'
         target = 'BIOS.Setup.1-1'

dracclient/tests/test_raid.py

@@ -265,7 +265,11 @@ class ClientRAIDManagementTestCase(base.BaseTest):
             expected_selectors, expected_properties,
             expected_return_value=utils.RET_SUCCESS)
 
-    def test_convert_physical_disks_fail(self, mock_requests):
+    @mock.patch.object(dracclient.client.WSManClient,
+                       'wait_until_idrac_is_ready', spec_set=True,
+                       autospec=True)
+    def test_convert_physical_disks_fail(self, mock_requests,
+                                         mock_wait_until_idrac_is_ready):
         mock_requests.post(
             'https://1.2.3.4:443/wsman',
             text=test_utils.RAIDInvocations[
@@ -334,7 +338,11 @@ class ClientRAIDManagementTestCase(base.BaseTest):
             expected_selectors, expected_properties,
             expected_return_value=utils.RET_SUCCESS)
 
-    def test_create_virtual_disk_fail(self, mock_requests):
+    @mock.patch.object(dracclient.client.WSManClient,
+                       'wait_until_idrac_is_ready', spec_set=True,
+                       autospec=True)
+    def test_create_virtual_disk_fail(self, mock_requests,
+                                      mock_wait_until_idrac_is_ready):
         mock_requests.post(
             'https://1.2.3.4:443/wsman',
             text=test_utils.RAIDInvocations[
@@ -422,7 +430,11 @@ class ClientRAIDManagementTestCase(base.BaseTest):
             expected_selectors, expected_properties,
             expected_return_value=utils.RET_SUCCESS)
 
-    def test_delete_virtual_disk_fail(self, mock_requests):
+    @mock.patch.object(dracclient.client.WSManClient,
+                       'wait_until_idrac_is_ready', spec_set=True,
+                       autospec=True)
+    def test_delete_virtual_disk_fail(self, mock_requests,
+                                      mock_wait_until_idrac_is_ready):
         mock_requests.post(
             'https://1.2.3.4:443/wsman',
             text=test_utils.RAIDInvocations[