openstack/python-dracclient
Code Issues Proposed changes

Fix immediate failure on SSL errors

Browse Source 
This patch adds retry logic to communication with the iDRAC so that
intermittent SSLErrors or ConnectionErrrors will not cause an immediate
failure of the operation.

Change-Id: Idc56e961699702eca734cba1da5e56cac0ad4832
Closes-Bug: 1691272
This commit is contained in:
Christopher Dearborn 2017-05-19 11:05:38 -04:00
parent f49efaa1bf
commit d6edaac2a1
3 changed files with 135 additions and 14 deletions

6
dracclient/client.py

@ -37,7 +37,7 @@ class DRACClient(object):
BIOS_DEVICE_FQDD = 'BIOS.Setup.1-1'
def __init__(self, host, username, password, port=443, path='/wsman',
protocol='https'):
protocol='https', retries=3, retry_delay=0):
"""Creates client object
:param host: hostname or IP of the DRAC interface
@ -46,9 +46,11 @@ class DRACClient(object):
:param port: port for accessing the DRAC interface
:param path: path for accessing the DRAC interface
:param protocol: protocol for accessing the DRAC interface
:param retries: number of resends to attempt on failure
:param retry_delay: number of seconds to wait between retries
"""
self.client = WSManClient(host, username, password, port, path,
protocol)
protocol, retries, retry_delay)
self._job_mgmt = job.JobManagement(self.client)
self._power_mgmt = bios.PowerManagement(self.client)
self._boot_mgmt = bios.BootManagement(self.client)

69
dracclient/tests/test_wsman.py

@ -17,6 +17,7 @@ import uuid
import lxml.etree
import lxml.objectify
import mock
import requests.exceptions
import requests_mock
from dracclient import exceptions
@ -108,6 +109,74 @@ class ClientTestCase(base.BaseTest):
self.assertEqual('yay!', resp.text)
@requests_mock.Mocker()
def test_invoke_with_ssl_errors(self, mock_requests):
mock_requests.post('https://1.2.3.4:443/wsman',
exc=requests.exceptions.SSLError)
self.assertRaises(exceptions.WSManRequestFailure,
self.client.invoke, 'http://resource', 'method',
{'selector': 'foo'}, {'property': 'bar'})
@requests_mock.Mocker()
def test_invoke_with_ssl_error_success(self, mock_requests):
expected_resp = '<result>yay!</result>'
mock_requests.post('https://1.2.3.4:443/wsman',
[{'exc': requests.exceptions.SSLError},
{'text': expected_resp}])
resp = self.client.invoke('http://resource', 'method',
{'selector': 'foo'}, {'property': 'bar'})
self.assertEqual('yay!', resp.text)
@requests_mock.Mocker()
def test_invoke_with_connection_errors(self, mock_requests):
mock_requests.post('https://1.2.3.4:443/wsman',
exc=requests.exceptions.ConnectionError)
self.assertRaises(exceptions.WSManRequestFailure,
self.client.invoke, 'http://resource', 'method',
{'selector': 'foo'}, {'property': 'bar'})
@requests_mock.Mocker()
def test_invoke_with_connection_error_success(self, mock_requests):
expected_resp = '<result>yay!</result>'
mock_requests.post('https://1.2.3.4:443/wsman',
[{'exc': requests.exceptions.ConnectionError},
{'text': expected_resp}])
resp = self.client.invoke('http://resource', 'method',
{'selector': 'foo'}, {'property': 'bar'})
self.assertEqual('yay!', resp.text)
@requests_mock.Mocker()
def test_invoke_with_unknown_error(self, mock_requests):
mock_requests.post('https://1.2.3.4:443/wsman',
exc=requests.exceptions.HTTPError)
self.assertRaises(exceptions.WSManRequestFailure,
self.client.invoke, 'http://resource', 'method',
{'selector': 'foo'}, {'property': 'bar'})
@requests_mock.Mocker()
@mock.patch('time.sleep', autospec=True)
def test_client_retry_delay(self, mock_requests, mock_ts):
retry_delay = 5
fake_endpoint = test_utils.FAKE_ENDPOINT.copy()
fake_endpoint['retry_delay'] = retry_delay
client = dracclient.wsman.Client(**fake_endpoint)
expected_resp = '<result>yay!</result>'
mock_requests.post('https://1.2.3.4:443/wsman',
[{'exc': requests.exceptions.SSLError},
{'text': expected_resp}])
resp = client.invoke('http://resource', 'method',
{'selector': 'foo'}, {'property': 'bar'})
self.assertEqual('yay!', resp.text)
mock_ts.assert_called_once_with(retry_delay)
class PayloadTestCase(base.BaseTest):

74
dracclient/wsman.py

@ -12,10 +12,10 @@
# under the License.
import logging
import time
import uuid
from lxml import etree as ElementTree
import requests
import requests.exceptions
from dracclient import exceptions
@ -41,13 +41,27 @@ class Client(object):
"""Simple client for talking over WSMan protocol."""
def __init__(self, host, username, password, port=443, path='/wsman',
protocol='https'):
protocol='https', retries=3, retry_delay=0):
"""Creates client object
:param host: hostname or IP of the DRAC interface
:param username: username for accessing the DRAC interface
:param password: password for accessing the DRAC interface
:param port: port for accessing the DRAC interface
:param path: path for accessing the DRAC interface
:param protocol: protocol for accessing the DRAC interface
:param retries: number of resends to attempt on failure
:param retry_delay: number of seconds to wait between retries
"""
self.host = host
self.username = username
self.password = password
self.port = port
self.path = path
self.protocol = protocol
self.retries = retries
self.retry_delay = retry_delay
self.endpoint = ('%(protocol)s://%(host)s:%(port)s%(path)s' % {
'protocol': self.protocol,
'host': self.host,
@ -58,16 +72,52 @@ class Client(object):
payload = payload.build()
LOG.debug('Sending request to %(endpoint)s: %(payload)s',
{'endpoint': self.endpoint, 'payload': payload})
try:
resp = requests.post(
self.endpoint,
auth=requests.auth.HTTPBasicAuth(self.username, self.password),
data=payload,
# TODO(ifarkas): enable cert verification
verify=False)
except requests.exceptions.RequestException:
LOG.exception('Request failed')
raise exceptions.WSManRequestFailure()
num_tries = 1
while num_tries <= self.retries:
try:
resp = requests.post(
self.endpoint,
auth=requests.auth.HTTPBasicAuth(self.username,
self.password),
data=payload,
# TODO(ifarkas): enable cert verification
verify=False)
break
except (requests.exceptions.ConnectionError,
requests.exceptions.SSLError) as ex:
error_msg = "A {error_type} error occurred while " \
" communicating with {host}, attempt {num_tries} of " \
"{retries}".format(
error_type=type(ex).__name__,
host=self.host,
num_tries=num_tries,
retries=self.retries)
if num_tries == self.retries:
LOG.error(error_msg)
raise exceptions.WSManRequestFailure(
"A {error_type} error occurred while communicating "
"with {host}: {error}".format(
error_type=type(ex).__name__,
host=self.host,
error=ex))
else:
LOG.warning(error_msg)
num_tries += 1
if self.retry_delay > 0 and num_tries <= self.retries:
time.sleep(self.retry_delay)
except requests.exceptions.RequestException as ex:
error_msg = "A {error_type} error occurred while " \
"communicating with {host}: {error}".format(
error_type=type(ex).__name__,
host=self.host,
error=ex)
LOG.error(error_msg)
raise exceptions.WSManRequestFailure(error_msg)
LOG.debug('Received response from %(endpoint)s: %(payload)s',
{'endpoint': self.endpoint, 'payload': resp.content})