From 683feaef9aa34f65e9ff7cfd93880d3b9e91e046 Mon Sep 17 00:00:00 2001 From: zengyingzhe Date: Fri, 10 Nov 2017 14:27:25 +0800 Subject: [PATCH] Utilize requests lib for Huawei storage connection For the latest Python 2.7 release, urllib uses the SSL certification while launching URL connection by default, which causes Huawei driver failed to connect backend storage because it doesn't support SSL certification. This patch fixes this issue by updating Huawei driver logic to use requests lib for Huawei storage connection, and specifying no use of SSL certification. Change-Id: Ia761819a352163176d353f61682cf47a1f429966 Closes-Bug: #1733451 --- manila/share/drivers/huawei/v3/helper.py | 90 ++++++++++--------- .../share/drivers/huawei/test_huawei_nas.py | 63 ++++++++++++- ...utilize-requests-lib-67f2c4e7ae0d2efa.yaml | 9 ++ 3 files changed, 118 insertions(+), 44 deletions(-) create mode 100644 releasenotes/notes/Huawei-driver-utilize-requests-lib-67f2c4e7ae0d2efa.yaml diff --git a/manila/share/drivers/huawei/v3/helper.py b/manila/share/drivers/huawei/v3/helper.py index afd2bebf3e..5eec41f576 100644 --- a/manila/share/drivers/huawei/v3/helper.py +++ b/manila/share/drivers/huawei/v3/helper.py @@ -15,14 +15,13 @@ import base64 import copy +import requests import time from xml.etree import ElementTree as ET from oslo_log import log from oslo_serialization import jsonutils import six -from six.moves import http_cookiejar -from six.moves.urllib import request as urlreq # pylint: disable=E0611 from manila import exception from manila.i18n import _ @@ -37,18 +36,23 @@ class RestHelper(object): def __init__(self, configuration): self.configuration = configuration - self.init_http_head() + self.url = None + self.session = None + + requests.packages.urllib3.disable_warnings( + requests.packages.urllib3.exceptions.InsecureRequestWarning) + requests.packages.urllib3.disable_warnings( + requests.packages.urllib3.exceptions.InsecurePlatformWarning) def init_http_head(self): - self.cookie = http_cookiejar.CookieJar() self.url = None - self.headers = { + self.session = requests.Session() + self.session.headers.update({ "Connection": "keep-alive", - "Content-Type": "application/json", - } + "Content-Type": "application/json"}) + self.session.verify = False - def do_call(self, url, data=None, method=None, - calltimeout=constants.SOCKET_TIMEOUT): + def do_call(self, url, data, method, calltimeout=constants.SOCKET_TIMEOUT): """Send requests to server. Send HTTPS call, get response in JSON. @@ -56,40 +60,41 @@ class RestHelper(object): """ if self.url: url = self.url + url - if "xx/sessions" not in url: - LOG.debug('Request URL: %(url)s\n' - 'Call Method: %(method)s\n' - 'Request Data: %(data)s\n', - {'url': url, - 'method': method, - 'data': data}) - opener = urlreq.build_opener(urlreq.HTTPCookieProcessor(self.cookie)) - urlreq.install_opener(opener) - result = None + + LOG.debug('Request URL: %(url)s\n' + 'Call Method: %(method)s\n' + 'Request Data: %(data)s\n', + {'url': url, + 'method': method, + 'data': data}) + + kwargs = {'timeout': calltimeout} + if data: + kwargs['data'] = data + + if method in ('POST', 'PUT', 'GET', 'DELETE'): + func = getattr(self.session, method.lower()) + else: + msg = _("Request method %s is invalid.") % method + LOG.error(msg) + raise exception.ShareBackendException(msg=msg) try: - req = urlreq.Request(url, data, self.headers) - if method: - req.get_method = lambda: method - res_temp = urlreq.urlopen(req, timeout=calltimeout) - res = res_temp.read().decode("utf-8") - - LOG.debug('Response Data: %(res)s.', {'res': res}) - + res = func(url, **kwargs) except Exception as err: LOG.error('\nBad response from server: %(url)s.' ' Error: %(err)s', {'url': url, 'err': err}) - res = ('{"error":{"code":%s,' - '"description":"Connect server error"}}' - % constants.ERROR_CONNECT_TO_SERVER) + return {"error": {"code": constants.ERROR_CONNECT_TO_SERVER, + "description": "Connect server error"}} try: - result = jsonutils.loads(res) - except Exception as err: - err_msg = (_('JSON transfer error: %s.') % err) - LOG.error(err_msg) - raise exception.InvalidInput(reason=err_msg) + res.raise_for_status() + except requests.HTTPError as exc: + return {"error": {"code": exc.response.status_code, + "description": six.text_type(exc)}} + result = res.json() + LOG.debug('Response Data: %s', result) return result def login(self): @@ -104,7 +109,7 @@ class RestHelper(object): "password": login_info['UserPassword'], "scope": "0"}) self.init_http_head() - result = self.do_call(url, data, + result = self.do_call(url, data, 'POST', calltimeout=constants.LOGIN_SOCKET_TIMEOUT) if((result['error']['code'] != 0) @@ -113,11 +118,10 @@ class RestHelper(object): LOG.error("Login to %s failed, try another.", item_url) continue - LOG.debug('Login success: %(url)s\n', - {'url': item_url}) + LOG.debug('Login success: %(url)s\n', {'url': item_url}) deviceid = result['data']['deviceid'] self.url = item_url + deviceid - self.headers['iBaseToken'] = result['data']['iBaseToken'] + self.session.headers['iBaseToken'] = result['data']['iBaseToken'] break if deviceid is None: @@ -128,7 +132,7 @@ class RestHelper(object): return deviceid @utils.synchronized('huawei_manila') - def call(self, url, data=None, method=None): + def call(self, url, data, method): """Send requests to server. If fail, try another RestURL. @@ -155,7 +159,7 @@ class RestHelper(object): """Create file system.""" url = "/filesystem" data = jsonutils.dumps(fs_param) - result = self.call(url, data) + result = self.call(url, data, 'POST') msg = 'Create filesystem error.' self._assert_rest_result(result, msg) @@ -353,7 +357,7 @@ class RestHelper(object): def _find_all_pool_info(self): url = "/storagepool" - result = self.call(url, None) + result = self.call(url, None, "GET") msg = "Query resource pool error." self._assert_rest_result(result, msg) @@ -971,7 +975,7 @@ class RestHelper(object): data = jsonutils.dumps(mergedata) url = "/ioclass" - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Create QoS policy error.')) return result['data']['ID'] diff --git a/manila/tests/share/drivers/huawei/test_huawei_nas.py b/manila/tests/share/drivers/huawei/test_huawei_nas.py index b6e74b202f..c8271850b2 100644 --- a/manila/tests/share/drivers/huawei/test_huawei_nas.py +++ b/manila/tests/share/drivers/huawei/test_huawei_nas.py @@ -17,6 +17,7 @@ import os +import requests import shutil import six import tempfile @@ -335,7 +336,7 @@ class FakeHuaweiNasHelper(helper.RestHelper): def _change_file_mode(self, filepath): pass - def do_call(self, url, data=None, method=None, calltimeout=4): + def do_call(self, url, data, method, calltimeout=4): url = url.replace('http://100.115.10.69:8082/deviceManager/rest', '') url = url.replace('/210235G7J20000000000/', '') @@ -4575,3 +4576,63 @@ class HuaweiShareDriverTestCase(test.TestCase): self.assertEqual(expect_username, result['UserName']) self.assertEqual(expect_password, result['UserPassword']) ET.parse.assert_called_once_with(self.fake_conf_file) + + +@ddt.ddt +class HuaweiDriverHelperTestCase(test.TestCase): + def setUp(self): + super(HuaweiDriverHelperTestCase, self).setUp() + self.helper = helper.RestHelper(None) + + def test_init_http_head(self): + self.helper.init_http_head() + self.assertIsNone(self.helper.url) + self.assertFalse(self.helper.session.verify) + self.assertEqual("keep-alive", + self.helper.session.headers["Connection"]) + self.assertEqual("application/json", + self.helper.session.headers["Content-Type"]) + + @ddt.data(('fake_data', 'POST'), + (None, 'POST'), + (None, 'PUT'), + (None, 'GET'), + ('fake_data', 'PUT'), + (None, 'DELETE'), + ) + @ddt.unpack + def test_do_call_with_valid_method(self, data, method): + self.helper.init_http_head() + + mocker = self.mock_object(self.helper.session, method.lower()) + self.helper.do_call("fake-rest-url", data, method) + + kwargs = {'timeout': constants.SOCKET_TIMEOUT} + if data: + kwargs['data'] = data + mocker.assert_called_once_with("fake-rest-url", **kwargs) + + def test_do_call_with_invalid_method(self): + self.assertRaises(exception.ShareBackendException, + self.helper.do_call, + "fake-rest-url", None, 'fake-method') + + def test_do_call_with_http_error(self): + self.helper.init_http_head() + + fake_res = requests.Response() + fake_res.reason = 'something wrong' + fake_res.status_code = 500 + fake_res.url = "fake-rest-url" + + self.mock_object(self.helper.session, 'post', + mock.Mock(return_value=fake_res)) + res = self.helper.do_call("fake-rest-url", None, 'POST') + + expected = { + "error": { + "code": 500, + "description": '500 Server Error: something wrong for ' + 'url: fake-rest-url'} + } + self.assertDictEqual(expected, res) diff --git a/releasenotes/notes/Huawei-driver-utilize-requests-lib-67f2c4e7ae0d2efa.yaml b/releasenotes/notes/Huawei-driver-utilize-requests-lib-67f2c4e7ae0d2efa.yaml new file mode 100644 index 0000000000..21d55c09f2 --- /dev/null +++ b/releasenotes/notes/Huawei-driver-utilize-requests-lib-67f2c4e7ae0d2efa.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + For the latest Python 2.7 release, urllib uses the SSL certification + while launching URL connection by default, which causes Huawei driver + failed to connect backend storage because it doesn't support SSL + certification. + Utilize the requests lib for Huawei driver instead, and set no SSL + certification for backend storage connection.