[NetApp] Fix HTTPS connection for python 3.7

With python3.7, the eventlet is breaking the ssl.py, so the https
is not working. This patch fixes it by changing the request library
(urllib by requests), the new library can be built over the
pyopenssl.py instead of ssl.py.

Closes-Bug: #1878993
Change-Id: I9c0b1f332ead25634f3dc3aebfdc8b51dfbc4178

manila/share/drivers/netapp/dataontap/client/api.py

@@ -23,8 +23,9 @@ import re
 
 from lxml import etree
 from oslo_log import log
+import requests
+from requests import auth
 import six
-from six.moves import urllib
 
 from manila import exception
 from manila.i18n import _
@@ -61,6 +62,7 @@ class NaServer(object):
 
     TRANSPORT_TYPE_HTTP = 'http'
     TRANSPORT_TYPE_HTTPS = 'https'
+    SSL_CERT_DEFAULT = "/etc/ssl/certs/"
     SERVER_TYPE_FILER = 'filer'
     SERVER_TYPE_DFM = 'dfm'
     URL_FILER = 'servlets/netapp.servlets.admin.XMLrequest_filer'
@@ -232,8 +234,8 @@ class NaServer(object):
         """Invoke the API on the server."""
         if na_element and not isinstance(na_element, NaElement):
             ValueError('NaElement must be supplied to invoke API')
-        request, request_element = self._create_request(na_element,
+        request_element = self._create_request(na_element, enable_tunneling)
-                                                        enable_tunneling)
+        request_d = request_element.to_string()
 
         api_name = na_element.get_name()
         api_name_matches_regex = (re.match(self._api_trace_pattern, api_name)
@@ -242,23 +244,26 @@ class NaServer(object):
         if self._trace and api_name_matches_regex:
             LOG.debug("Request: %s", request_element.to_string(pretty=True))
 
-        if (not hasattr(self, '_opener') or not self._opener
+        if (not hasattr(self, '_session') or not self._session
                 or self._refresh_conn):
-            self._build_opener()
+            self._build_session()
         try:
             if hasattr(self, '_timeout'):
-                response = self._opener.open(request, timeout=self._timeout)
+                response = self._session.post(
+                    self._get_url(), data=request_d, timeout=self._timeout)
             else:
-                response = self._opener.open(request)
+                response = self._session.post(
-        except urllib.error.HTTPError as e:
+                    self._get_url(), data=request_d)
-            raise NaApiError(e.code, e.msg)
+        except requests.HTTPError as e:
-        except urllib.error.URLError as e:
+            raise NaApiError(e.errno, e.strerror)
+        except requests.URLRequired as e:
             raise exception.StorageCommunicationException(six.text_type(e))
         except Exception as e:
             raise NaApiError(message=e)
 
-        response_xml = response.read()
+        response_xml = response.text
-        response_element = self._get_result(response_xml)
+        response_element = self._get_result(
+            bytes(bytearray(response_xml, encoding='utf-8')))
 
         if self._trace and api_name_matches_regex:
             LOG.debug("Response: %s", response_element.to_string(pretty=True))
@@ -296,11 +301,7 @@ class NaServer(object):
         if enable_tunneling:
             self._enable_tunnel_request(netapp_elem)
         netapp_elem.add_child_elem(na_element)
-        request_d = netapp_elem.to_string()
+        return netapp_elem
-        request = urllib.request.Request(
-            self._get_url(), data=request_d,
-            headers={'Content-Type': 'text/xml', 'charset': 'utf-8'})
-        return request, netapp_elem
 
     def _enable_tunnel_request(self, netapp_elem):
         """Enables vserver or vfiler tunneling."""
@@ -341,20 +342,20 @@ class NaServer(object):
             host = '[%s]' % host
         return '%s://%s:%s/%s' % (self._protocol, host, self._port, self._url)
 
-    def _build_opener(self):
+    def _build_session(self):
         if self._auth_style == NaServer.STYLE_LOGIN_PASSWORD:
             auth_handler = self._create_basic_auth_handler()
         else:
             auth_handler = self._create_certificate_auth_handler()
-        opener = urllib.request.build_opener(auth_handler)
+
-        self._opener = opener
+        self._session = requests.Session()
+        self._session.auth = auth_handler
+        self._session.verify = NaServer.SSL_CERT_DEFAULT
+        self._session.headers = {
+            'Content-Type': 'text/xml', 'charset': 'utf-8'}
 
     def _create_basic_auth_handler(self):
-        password_man = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+        return auth.HTTPBasicAuth(self._username, self._password)
-        password_man.add_password(None, self._get_url(), self._username,
-                                  self._password)
-        auth_handler = urllib.request.HTTPBasicAuthHandler(password_man)
-        return auth_handler
 
     def _create_certificate_auth_handler(self):
         raise NotImplementedError()

manila/tests/share/drivers/netapp/dataontap/client/fakes.py

@@ -15,7 +15,7 @@
 from unittest import mock
 
 from lxml import etree
-from six.moves import urllib
+import requests
 
 from manila.share.drivers.netapp.dataontap.client import api
 
@@ -2638,7 +2638,7 @@ FAKE_RESULT_API_ERRNO_VALID.add_attr('errno', '14956')
 FAKE_RESULT_SUCCESS = api.NaElement('result')
 FAKE_RESULT_SUCCESS.add_attr('status', 'passed')
 
-FAKE_HTTP_OPENER = urllib.request.build_opener()
+FAKE_HTTP_SESSION = requests.Session()
 
 FAKE_MANAGE_VOLUME = {
     'aggregate': SHARE_AGGREGATE_NAME,

manila/tests/share/drivers/netapp/dataontap/client/test_api.py

@@ -22,7 +22,7 @@ Tests for NetApp API layer
 from unittest import mock
 
 import ddt
-from six.moves import urllib
+import requests
 
 from manila import exception
 from manila.share.drivers.netapp.dataontap.client import api
@@ -190,14 +190,12 @@ class NetAppApiServerTests(test.TestCase):
         """Tests handling of HTTPError"""
         na_element = fake.FAKE_NA_ELEMENT
         self.mock_object(self.root, '_create_request', mock.Mock(
-            return_value=('abc', fake.FAKE_NA_ELEMENT)))
+            return_value=fake.FAKE_NA_ELEMENT))
         self.mock_object(api, 'LOG')
-        self.root._opener = fake.FAKE_HTTP_OPENER
+        self.root._session = fake.FAKE_HTTP_SESSION
-        self.mock_object(self.root, '_build_opener')
+        self.mock_object(self.root, '_build_session')
-        self.mock_object(self.root._opener, 'open', mock.Mock(
+        self.mock_object(self.root._session, 'post', mock.Mock(
-            side_effect=urllib.error.HTTPError(url='', hdrs='',
+            side_effect=requests.HTTPError()))
-                                               fp=None, code='401',
-                                               msg='httperror')))
 
         self.assertRaises(api.NaApiError, self.root.invoke_elem,
                           na_element)
@@ -206,12 +204,12 @@ class NetAppApiServerTests(test.TestCase):
         """Tests handling of URLError"""
         na_element = fake.FAKE_NA_ELEMENT
         self.mock_object(self.root, '_create_request', mock.Mock(
-            return_value=('abc', fake.FAKE_NA_ELEMENT)))
+            return_value=fake.FAKE_NA_ELEMENT))
         self.mock_object(api, 'LOG')
-        self.root._opener = fake.FAKE_HTTP_OPENER
+        self.root._session = fake.FAKE_HTTP_SESSION
-        self.mock_object(self.root, '_build_opener')
+        self.mock_object(self.root, '_build_session')
-        self.mock_object(self.root._opener, 'open', mock.Mock(
+        self.mock_object(self.root._session, 'post', mock.Mock(
-            side_effect=urllib.error.URLError(reason='urlerror')))
+            side_effect=requests.URLRequired()))
 
         self.assertRaises(exception.StorageCommunicationException,
                           self.root.invoke_elem,
@@ -221,11 +219,11 @@ class NetAppApiServerTests(test.TestCase):
         """Tests handling of Unknown Exception"""
         na_element = fake.FAKE_NA_ELEMENT
         self.mock_object(self.root, '_create_request', mock.Mock(
-            return_value=('abc', fake.FAKE_NA_ELEMENT)))
+            return_value=fake.FAKE_NA_ELEMENT))
         self.mock_object(api, 'LOG')
-        self.root._opener = fake.FAKE_HTTP_OPENER
+        self.root._session = fake.FAKE_HTTP_SESSION
-        self.mock_object(self.root, '_build_opener')
+        self.mock_object(self.root, '_build_session')
-        self.mock_object(self.root._opener, 'open', mock.Mock(
+        self.mock_object(self.root._session, 'post', mock.Mock(
             side_effect=Exception))
 
         exception = self.assertRaises(api.NaApiError, self.root.invoke_elem,
@@ -247,15 +245,18 @@ class NetAppApiServerTests(test.TestCase):
         self.root._trace = trace_enabled
         self.root._api_trace_pattern = trace_pattern
         self.mock_object(self.root, '_create_request', mock.Mock(
-            return_value=('abc', fake.FAKE_NA_ELEMENT)))
+            return_value=fake.FAKE_NA_ELEMENT))
         self.mock_object(api, 'LOG')
-        self.root._opener = fake.FAKE_HTTP_OPENER
+        self.root._session = fake.FAKE_HTTP_SESSION
-        self.mock_object(self.root, '_build_opener')
+        self.mock_object(self.root, '_build_session')
         self.mock_object(self.root, '_get_result', mock.Mock(
             return_value=fake.FAKE_NA_ELEMENT))
-        opener_mock = self.mock_object(
+
-            self.root._opener, 'open', mock.Mock())
+        response = mock.Mock()
-        opener_mock.read.side_effect = ['resp1', 'resp2']
+        response.text = 'res1'
+        self.mock_object(
+            self.root._session, 'post', mock.Mock(
+                return_value=response))
 
         self.root.invoke_elem(na_element)
 

releasenotes/notes/bug-1878993-netapp-fix-https-3eddf9eb5b762f3a.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixed `bug #1878993 <https://bugs.launchpad.net/manila/+bug/1878993>`_
+    that caused a failure on HTTPS connections within NetApp backend using
+    python 3.7.