Utilize requests lib for Huawei storage connection

For the latest Python 2.7 release, urllib uses the SSL certification
while launching URL connection by default, which causes Huawei driver
failed to connect backend storage because it doesn't support SSL
certification.
This patch fixes this issue by updating Huawei driver logic to use
requests lib for Huawei storage connection, and specifying no use of
SSL certification.

Change-Id: Ia761819a352163176d353f61682cf47a1f429966
Closes-Bug: #1733451

manila/share/drivers/huawei/v3/helper.py

@@ -15,14 +15,13 @@
 
 import base64
 import copy
+import requests
 import time
 from xml.etree import ElementTree as ET
 
 from oslo_log import log
 from oslo_serialization import jsonutils
 import six
-from six.moves import http_cookiejar
-from six.moves.urllib import request as urlreq  # pylint: disable=E0611
 
 from manila import exception
 from manila.i18n import _
@@ -37,18 +36,23 @@ class RestHelper(object):
 
     def __init__(self, configuration):
         self.configuration = configuration
-        self.init_http_head()
+        self.url = None
+        self.session = None
+
+        requests.packages.urllib3.disable_warnings(
+            requests.packages.urllib3.exceptions.InsecureRequestWarning)
+        requests.packages.urllib3.disable_warnings(
+            requests.packages.urllib3.exceptions.InsecurePlatformWarning)
 
     def init_http_head(self):
-        self.cookie = http_cookiejar.CookieJar()
         self.url = None
-        self.headers = {
+        self.session = requests.Session()
+        self.session.headers.update({
             "Connection": "keep-alive",
-            "Content-Type": "application/json",
+            "Content-Type": "application/json"})
-        }
+        self.session.verify = False
 
-    def do_call(self, url, data=None, method=None,
+    def do_call(self, url, data, method, calltimeout=constants.SOCKET_TIMEOUT):
-                calltimeout=constants.SOCKET_TIMEOUT):
         """Send requests to server.
 
         Send HTTPS call, get response in JSON.
@@ -56,40 +60,41 @@ class RestHelper(object):
         """
         if self.url:
             url = self.url + url
-        if "xx/sessions" not in url:
+
-            LOG.debug('Request URL: %(url)s\n'
+        LOG.debug('Request URL: %(url)s\n'
-                      'Call Method: %(method)s\n'
+                  'Call Method: %(method)s\n'
-                      'Request Data: %(data)s\n',
+                  'Request Data: %(data)s\n',
-                      {'url': url,
+                  {'url': url,
-                       'method': method,
+                   'method': method,
-                       'data': data})
+                   'data': data})
-        opener = urlreq.build_opener(urlreq.HTTPCookieProcessor(self.cookie))
+
-        urlreq.install_opener(opener)
+        kwargs = {'timeout': calltimeout}
-        result = None
+        if data:
+            kwargs['data'] = data
+
+        if method in ('POST', 'PUT', 'GET', 'DELETE'):
+            func = getattr(self.session, method.lower())
+        else:
+            msg = _("Request method %s is invalid.") % method
+            LOG.error(msg)
+            raise exception.ShareBackendException(msg=msg)
 
         try:
-            req = urlreq.Request(url, data, self.headers)
+            res = func(url, **kwargs)
-            if method:
-                req.get_method = lambda: method
-            res_temp = urlreq.urlopen(req, timeout=calltimeout)
-            res = res_temp.read().decode("utf-8")
-
-            LOG.debug('Response Data: %(res)s.', {'res': res})
-
         except Exception as err:
             LOG.error('\nBad response from server: %(url)s.'
                       ' Error: %(err)s', {'url': url, 'err': err})
-            res = ('{"error":{"code":%s,'
+            return {"error": {"code": constants.ERROR_CONNECT_TO_SERVER,
-                   '"description":"Connect server error"}}'
+                              "description": "Connect server error"}}
-                   % constants.ERROR_CONNECT_TO_SERVER)
 
         try:
-            result = jsonutils.loads(res)
+            res.raise_for_status()
-        except Exception as err:
+        except requests.HTTPError as exc:
-            err_msg = (_('JSON transfer error: %s.') % err)
+            return {"error": {"code": exc.response.status_code,
-            LOG.error(err_msg)
+                              "description": six.text_type(exc)}}
-            raise exception.InvalidInput(reason=err_msg)
 
+        result = res.json()
+        LOG.debug('Response Data: %s', result)
         return result
 
     def login(self):
@@ -104,7 +109,7 @@ class RestHelper(object):
                                     "password": login_info['UserPassword'],
                                     "scope": "0"})
             self.init_http_head()
-            result = self.do_call(url, data,
+            result = self.do_call(url, data, 'POST',
                                   calltimeout=constants.LOGIN_SOCKET_TIMEOUT)
 
             if((result['error']['code'] != 0)
@@ -113,11 +118,10 @@ class RestHelper(object):
                 LOG.error("Login to %s failed, try another.", item_url)
                 continue
 
-            LOG.debug('Login success: %(url)s\n',
+            LOG.debug('Login success: %(url)s\n', {'url': item_url})
-                      {'url': item_url})
             deviceid = result['data']['deviceid']
             self.url = item_url + deviceid
-            self.headers['iBaseToken'] = result['data']['iBaseToken']
+            self.session.headers['iBaseToken'] = result['data']['iBaseToken']
             break
 
         if deviceid is None:
@@ -128,7 +132,7 @@ class RestHelper(object):
         return deviceid
 
     @utils.synchronized('huawei_manila')
-    def call(self, url, data=None, method=None):
+    def call(self, url, data, method):
         """Send requests to server.
 
         If fail, try another RestURL.
@@ -155,7 +159,7 @@ class RestHelper(object):
         """Create file system."""
         url = "/filesystem"
         data = jsonutils.dumps(fs_param)
-        result = self.call(url, data)
+        result = self.call(url, data, 'POST')
 
         msg = 'Create filesystem error.'
         self._assert_rest_result(result, msg)
@@ -353,7 +357,7 @@ class RestHelper(object):
 
     def _find_all_pool_info(self):
         url = "/storagepool"
-        result = self.call(url, None)
+        result = self.call(url, None, "GET")
 
         msg = "Query resource pool error."
         self._assert_rest_result(result, msg)
@@ -971,7 +975,7 @@ class RestHelper(object):
         data = jsonutils.dumps(mergedata)
         url = "/ioclass"
 
-        result = self.call(url, data)
+        result = self.call(url, data, 'POST')
         self._assert_rest_result(result, _('Create QoS policy error.'))
 
         return result['data']['ID']

manila/tests/share/drivers/huawei/test_huawei_nas.py

@@ -17,6 +17,7 @@
 
 
 import os
+import requests
 import shutil
 import six
 import tempfile
@@ -335,7 +336,7 @@ class FakeHuaweiNasHelper(helper.RestHelper):
     def _change_file_mode(self, filepath):
         pass
 
-    def do_call(self, url, data=None, method=None, calltimeout=4):
+    def do_call(self, url, data, method, calltimeout=4):
         url = url.replace('http://100.115.10.69:8082/deviceManager/rest', '')
         url = url.replace('/210235G7J20000000000/', '')
 
@@ -4575,3 +4576,63 @@ class HuaweiShareDriverTestCase(test.TestCase):
         self.assertEqual(expect_username, result['UserName'])
         self.assertEqual(expect_password, result['UserPassword'])
         ET.parse.assert_called_once_with(self.fake_conf_file)
+
+
+@ddt.ddt
+class HuaweiDriverHelperTestCase(test.TestCase):
+    def setUp(self):
+        super(HuaweiDriverHelperTestCase, self).setUp()
+        self.helper = helper.RestHelper(None)
+
+    def test_init_http_head(self):
+        self.helper.init_http_head()
+        self.assertIsNone(self.helper.url)
+        self.assertFalse(self.helper.session.verify)
+        self.assertEqual("keep-alive",
+                         self.helper.session.headers["Connection"])
+        self.assertEqual("application/json",
+                         self.helper.session.headers["Content-Type"])
+
+    @ddt.data(('fake_data', 'POST'),
+              (None, 'POST'),
+              (None, 'PUT'),
+              (None, 'GET'),
+              ('fake_data', 'PUT'),
+              (None, 'DELETE'),
+              )
+    @ddt.unpack
+    def test_do_call_with_valid_method(self, data, method):
+        self.helper.init_http_head()
+
+        mocker = self.mock_object(self.helper.session, method.lower())
+        self.helper.do_call("fake-rest-url", data, method)
+
+        kwargs = {'timeout': constants.SOCKET_TIMEOUT}
+        if data:
+            kwargs['data'] = data
+        mocker.assert_called_once_with("fake-rest-url", **kwargs)
+
+    def test_do_call_with_invalid_method(self):
+        self.assertRaises(exception.ShareBackendException,
+                          self.helper.do_call,
+                          "fake-rest-url", None, 'fake-method')
+
+    def test_do_call_with_http_error(self):
+        self.helper.init_http_head()
+
+        fake_res = requests.Response()
+        fake_res.reason = 'something wrong'
+        fake_res.status_code = 500
+        fake_res.url = "fake-rest-url"
+
+        self.mock_object(self.helper.session, 'post',
+                         mock.Mock(return_value=fake_res))
+        res = self.helper.do_call("fake-rest-url", None, 'POST')
+
+        expected = {
+            "error": {
+                "code": 500,
+                "description": '500 Server Error: something wrong for '
+                               'url: fake-rest-url'}
+        }
+        self.assertDictEqual(expected, res)

releasenotes/notes/Huawei-driver-utilize-requests-lib-67f2c4e7ae0d2efa.yaml

@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    For the latest Python 2.7 release, urllib uses the SSL certification
+    while launching URL connection by default, which causes Huawei driver
+    failed to connect backend storage because it doesn't support SSL
+    certification.
+    Utilize the requests lib for Huawei driver instead, and set no SSL
+    certification for backend storage connection.