Merge "Utilize requests lib for Huawei storage connection"

manila/share/drivers/huawei/v3/helper.py

@@ -15,14 +15,13 @@
 
 import base64
 import copy
+import requests
 import time
 from xml.etree import ElementTree as ET
 
 from oslo_log import log
 from oslo_serialization import jsonutils
 import six
-from six.moves import http_cookiejar
-from six.moves.urllib import request as urlreq  # pylint: disable=E0611
 
 from manila import exception
 from manila.i18n import _
@@ -37,18 +36,23 @@ class RestHelper(object):
 
     def __init__(self, configuration):
         self.configuration = configuration
-        self.init_http_head()
+        self.url = None
+        self.session = None
+
+        requests.packages.urllib3.disable_warnings(
+            requests.packages.urllib3.exceptions.InsecureRequestWarning)
+        requests.packages.urllib3.disable_warnings(
+            requests.packages.urllib3.exceptions.InsecurePlatformWarning)
 
     def init_http_head(self):
-        self.cookie = http_cookiejar.CookieJar()
         self.url = None
-        self.headers = {
+        self.session = requests.Session()
+        self.session.headers.update({
             "Connection": "keep-alive",
-            "Content-Type": "application/json",
+            "Content-Type": "application/json"})
-        }
+        self.session.verify = False
 
-    def do_call(self, url, data=None, method=None,
+    def do_call(self, url, data, method, calltimeout=constants.SOCKET_TIMEOUT):
-                calltimeout=constants.SOCKET_TIMEOUT):
         """Send requests to server.
 
         Send HTTPS call, get response in JSON.
@@ -56,40 +60,41 @@ class RestHelper(object):
         """
         if self.url:
             url = self.url + url
-        if "xx/sessions" not in url:
+
-            LOG.debug('Request URL: %(url)s\n'
+        LOG.debug('Request URL: %(url)s\n'
-                      'Call Method: %(method)s\n'
+                  'Call Method: %(method)s\n'
-                      'Request Data: %(data)s\n',
+                  'Request Data: %(data)s\n',
-                      {'url': url,
+                  {'url': url,
-                       'method': method,
+                   'method': method,
-                       'data': data})
+                   'data': data})
-        opener = urlreq.build_opener(urlreq.HTTPCookieProcessor(self.cookie))
+
-        urlreq.install_opener(opener)
+        kwargs = {'timeout': calltimeout}
-        result = None
+        if data:
+            kwargs['data'] = data
+
+        if method in ('POST', 'PUT', 'GET', 'DELETE'):
+            func = getattr(self.session, method.lower())
+        else:
+            msg = _("Request method %s is invalid.") % method
+            LOG.error(msg)
+            raise exception.ShareBackendException(msg=msg)
 
         try:
-            req = urlreq.Request(url, data, self.headers)
+            res = func(url, **kwargs)
-            if method:
-                req.get_method = lambda: method
-            res_temp = urlreq.urlopen(req, timeout=calltimeout)
-            res = res_temp.read().decode("utf-8")
-
-            LOG.debug('Response Data: %(res)s.', {'res': res})
-
         except Exception as err:
             LOG.error('\nBad response from server: %(url)s.'
                       ' Error: %(err)s', {'url': url, 'err': err})
-            res = ('{"error":{"code":%s,'
+            return {"error": {"code": constants.ERROR_CONNECT_TO_SERVER,
-                   '"description":"Connect server error"}}'
+                              "description": "Connect server error"}}
-                   % constants.ERROR_CONNECT_TO_SERVER)
 
         try:
-            result = jsonutils.loads(res)
+            res.raise_for_status()
-        except Exception as err:
+        except requests.HTTPError as exc:
-            err_msg = (_('JSON transfer error: %s.') % err)
+            return {"error": {"code": exc.response.status_code,
-            LOG.error(err_msg)
+                              "description": six.text_type(exc)}}
-            raise exception.InvalidInput(reason=err_msg)
 
+        result = res.json()
+        LOG.debug('Response Data: %s', result)
         return result
 
     def login(self):
@@ -104,7 +109,7 @@ class RestHelper(object):
                                     "password": login_info['UserPassword'],
                                     "scope": "0"})
             self.init_http_head()
-            result = self.do_call(url, data,
+            result = self.do_call(url, data, 'POST',
                                   calltimeout=constants.LOGIN_SOCKET_TIMEOUT)
 
             if((result['error']['code'] != 0)
@@ -113,11 +118,10 @@ class RestHelper(object):
                 LOG.error("Login to %s failed, try another.", item_url)
                 continue
 
-            LOG.debug('Login success: %(url)s\n',
+            LOG.debug('Login success: %(url)s\n', {'url': item_url})
-                      {'url': item_url})
             deviceid = result['data']['deviceid']
             self.url = item_url + deviceid
-            self.headers['iBaseToken'] = result['data']['iBaseToken']
+            self.session.headers['iBaseToken'] = result['data']['iBaseToken']
             break
 
         if deviceid is None:
@@ -128,7 +132,7 @@ class RestHelper(object):
         return deviceid
 
     @utils.synchronized('huawei_manila')
-    def call(self, url, data=None, method=None):
+    def call(self, url, data, method):
         """Send requests to server.
 
         If fail, try another RestURL.
@@ -155,7 +159,7 @@ class RestHelper(object):
         """Create file system."""
         url = "/filesystem"
         data = jsonutils.dumps(fs_param)
-        result = self.call(url, data)
+        result = self.call(url, data, 'POST')
 
         msg = 'Create filesystem error.'
         self._assert_rest_result(result, msg)
@@ -353,7 +357,7 @@ class RestHelper(object):
 
     def _find_all_pool_info(self):
         url = "/storagepool"
-        result = self.call(url, None)
+        result = self.call(url, None, "GET")
 
         msg = "Query resource pool error."
         self._assert_rest_result(result, msg)
@@ -971,7 +975,7 @@ class RestHelper(object):
         data = jsonutils.dumps(mergedata)
         url = "/ioclass"
 
-        result = self.call(url, data)
+        result = self.call(url, data, 'POST')
         self._assert_rest_result(result, _('Create QoS policy error.'))
 
         return result['data']['ID']

manila/tests/share/drivers/huawei/test_huawei_nas.py

@@ -17,6 +17,7 @@
 
 
 import os
+import requests
 import shutil
 import six
 import tempfile
@@ -335,7 +336,7 @@ class FakeHuaweiNasHelper(helper.RestHelper):
     def _change_file_mode(self, filepath):
         pass
 
-    def do_call(self, url, data=None, method=None, calltimeout=4):
+    def do_call(self, url, data, method, calltimeout=4):
         url = url.replace('http://100.115.10.69:8082/deviceManager/rest', '')
         url = url.replace('/210235G7J20000000000/', '')
 
@@ -4575,3 +4576,63 @@ class HuaweiShareDriverTestCase(test.TestCase):
         self.assertEqual(expect_username, result['UserName'])
         self.assertEqual(expect_password, result['UserPassword'])
         ET.parse.assert_called_once_with(self.fake_conf_file)
+
+
+@ddt.ddt
+class HuaweiDriverHelperTestCase(test.TestCase):
+    def setUp(self):
+        super(HuaweiDriverHelperTestCase, self).setUp()
+        self.helper = helper.RestHelper(None)
+
+    def test_init_http_head(self):
+        self.helper.init_http_head()
+        self.assertIsNone(self.helper.url)
+        self.assertFalse(self.helper.session.verify)
+        self.assertEqual("keep-alive",
+                         self.helper.session.headers["Connection"])
+        self.assertEqual("application/json",
+                         self.helper.session.headers["Content-Type"])
+
+    @ddt.data(('fake_data', 'POST'),
+              (None, 'POST'),
+              (None, 'PUT'),
+              (None, 'GET'),
+              ('fake_data', 'PUT'),
+              (None, 'DELETE'),
+              )
+    @ddt.unpack
+    def test_do_call_with_valid_method(self, data, method):
+        self.helper.init_http_head()
+
+        mocker = self.mock_object(self.helper.session, method.lower())
+        self.helper.do_call("fake-rest-url", data, method)
+
+        kwargs = {'timeout': constants.SOCKET_TIMEOUT}
+        if data:
+            kwargs['data'] = data
+        mocker.assert_called_once_with("fake-rest-url", **kwargs)
+
+    def test_do_call_with_invalid_method(self):
+        self.assertRaises(exception.ShareBackendException,
+                          self.helper.do_call,
+                          "fake-rest-url", None, 'fake-method')
+
+    def test_do_call_with_http_error(self):
+        self.helper.init_http_head()
+
+        fake_res = requests.Response()
+        fake_res.reason = 'something wrong'
+        fake_res.status_code = 500
+        fake_res.url = "fake-rest-url"
+
+        self.mock_object(self.helper.session, 'post',
+                         mock.Mock(return_value=fake_res))
+        res = self.helper.do_call("fake-rest-url", None, 'POST')
+
+        expected = {
+            "error": {
+                "code": 500,
+                "description": '500 Server Error: something wrong for '
+                               'url: fake-rest-url'}
+        }
+        self.assertDictEqual(expected, res)

releasenotes/notes/Huawei-driver-utilize-requests-lib-67f2c4e7ae0d2efa.yaml

@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    For the latest Python 2.7 release, urllib uses the SSL certification
+    while launching URL connection by default, which causes Huawei driver
+    failed to connect backend storage because it doesn't support SSL
+    certification.
+    Utilize the requests lib for Huawei driver instead, and set no SSL
+    certification for backend storage connection.