Merge "[NetApp] Fix HTTPS connection for python 3.7"

manila/share/drivers/netapp/dataontap/client/api.py

@@ -23,8 +23,9 @@ import re
 
 from lxml import etree
 from oslo_log import log
+import requests
+from requests import auth
 import six
-from six.moves import urllib
 
 from manila import exception
 from manila.i18n import _
@@ -61,6 +62,7 @@ class NaServer(object):
 
     TRANSPORT_TYPE_HTTP = 'http'
     TRANSPORT_TYPE_HTTPS = 'https'
+    SSL_CERT_DEFAULT = "/etc/ssl/certs/"
     SERVER_TYPE_FILER = 'filer'
     SERVER_TYPE_DFM = 'dfm'
     URL_FILER = 'servlets/netapp.servlets.admin.XMLrequest_filer'
@@ -232,8 +234,8 @@ class NaServer(object):
         """Invoke the API on the server."""
         if na_element and not isinstance(na_element, NaElement):
             ValueError('NaElement must be supplied to invoke API')
-        request, request_element = self._create_request(na_element,
+        request_element = self._create_request(na_element, enable_tunneling)
-                                                        enable_tunneling)
+        request_d = request_element.to_string()
 
         api_name = na_element.get_name()
         api_name_matches_regex = (re.match(self._api_trace_pattern, api_name)
@@ -242,23 +244,26 @@ class NaServer(object):
         if self._trace and api_name_matches_regex:
             LOG.debug("Request: %s", request_element.to_string(pretty=True))
 
-        if (not hasattr(self, '_opener') or not self._opener
+        if (not hasattr(self, '_session') or not self._session
                 or self._refresh_conn):
-            self._build_opener()
+            self._build_session()
         try:
             if hasattr(self, '_timeout'):
-                response = self._opener.open(request, timeout=self._timeout)
+                response = self._session.post(
+                    self._get_url(), data=request_d, timeout=self._timeout)
             else:
-                response = self._opener.open(request)
+                response = self._session.post(
-        except urllib.error.HTTPError as e:
+                    self._get_url(), data=request_d)
-            raise NaApiError(e.code, e.msg)
+        except requests.HTTPError as e:
-        except urllib.error.URLError as e:
+            raise NaApiError(e.errno, e.strerror)
+        except requests.URLRequired as e:
             raise exception.StorageCommunicationException(six.text_type(e))
         except Exception as e:
             raise NaApiError(message=e)
 
-        response_xml = response.read()
+        response_xml = response.text
-        response_element = self._get_result(response_xml)
+        response_element = self._get_result(
+            bytes(bytearray(response_xml, encoding='utf-8')))
 
         if self._trace and api_name_matches_regex:
             LOG.debug("Response: %s", response_element.to_string(pretty=True))
@@ -296,11 +301,7 @@ class NaServer(object):
         if enable_tunneling:
             self._enable_tunnel_request(netapp_elem)
         netapp_elem.add_child_elem(na_element)
-        request_d = netapp_elem.to_string()
+        return netapp_elem
-        request = urllib.request.Request(
-            self._get_url(), data=request_d,
-            headers={'Content-Type': 'text/xml', 'charset': 'utf-8'})
-        return request, netapp_elem
 
     def _enable_tunnel_request(self, netapp_elem):
         """Enables vserver or vfiler tunneling."""
@@ -341,20 +342,20 @@ class NaServer(object):
             host = '[%s]' % host
         return '%s://%s:%s/%s' % (self._protocol, host, self._port, self._url)
 
-    def _build_opener(self):
+    def _build_session(self):
         if self._auth_style == NaServer.STYLE_LOGIN_PASSWORD:
             auth_handler = self._create_basic_auth_handler()
         else:
             auth_handler = self._create_certificate_auth_handler()
-        opener = urllib.request.build_opener(auth_handler)
+
-        self._opener = opener
+        self._session = requests.Session()
+        self._session.auth = auth_handler
+        self._session.verify = NaServer.SSL_CERT_DEFAULT
+        self._session.headers = {
+            'Content-Type': 'text/xml', 'charset': 'utf-8'}
 
     def _create_basic_auth_handler(self):
-        password_man = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+        return auth.HTTPBasicAuth(self._username, self._password)
-        password_man.add_password(None, self._get_url(), self._username,
-                                  self._password)
-        auth_handler = urllib.request.HTTPBasicAuthHandler(password_man)
-        return auth_handler
 
     def _create_certificate_auth_handler(self):
         raise NotImplementedError()

manila/tests/share/drivers/netapp/dataontap/client/fakes.py

@@ -15,7 +15,7 @@
 from unittest import mock
 
 from lxml import etree
-from six.moves import urllib
+import requests
 
 from manila.share.drivers.netapp.dataontap.client import api
 
@@ -2638,7 +2638,7 @@ FAKE_RESULT_API_ERRNO_VALID.add_attr('errno', '14956')
 FAKE_RESULT_SUCCESS = api.NaElement('result')
 FAKE_RESULT_SUCCESS.add_attr('status', 'passed')
 
-FAKE_HTTP_OPENER = urllib.request.build_opener()
+FAKE_HTTP_SESSION = requests.Session()
 
 FAKE_MANAGE_VOLUME = {
     'aggregate': SHARE_AGGREGATE_NAME,

manila/tests/share/drivers/netapp/dataontap/client/test_api.py

@@ -22,7 +22,7 @@ Tests for NetApp API layer
 from unittest import mock
 
 import ddt
-from six.moves import urllib
+import requests
 
 from manila import exception
 from manila.share.drivers.netapp.dataontap.client import api
@@ -190,14 +190,12 @@ class NetAppApiServerTests(test.TestCase):
         """Tests handling of HTTPError"""
         na_element = fake.FAKE_NA_ELEMENT
         self.mock_object(self.root, '_create_request', mock.Mock(
-            return_value=('abc', fake.FAKE_NA_ELEMENT)))
+            return_value=fake.FAKE_NA_ELEMENT))
         self.mock_object(api, 'LOG')
-        self.root._opener = fake.FAKE_HTTP_OPENER
+        self.root._session = fake.FAKE_HTTP_SESSION
-        self.mock_object(self.root, '_build_opener')
+        self.mock_object(self.root, '_build_session')
-        self.mock_object(self.root._opener, 'open', mock.Mock(
+        self.mock_object(self.root._session, 'post', mock.Mock(
-            side_effect=urllib.error.HTTPError(url='', hdrs='',
+            side_effect=requests.HTTPError()))
-                                               fp=None, code='401',
-                                               msg='httperror')))
 
         self.assertRaises(api.NaApiError, self.root.invoke_elem,
                           na_element)
@@ -206,12 +204,12 @@ class NetAppApiServerTests(test.TestCase):
         """Tests handling of URLError"""
         na_element = fake.FAKE_NA_ELEMENT
         self.mock_object(self.root, '_create_request', mock.Mock(
-            return_value=('abc', fake.FAKE_NA_ELEMENT)))
+            return_value=fake.FAKE_NA_ELEMENT))
         self.mock_object(api, 'LOG')
-        self.root._opener = fake.FAKE_HTTP_OPENER
+        self.root._session = fake.FAKE_HTTP_SESSION
-        self.mock_object(self.root, '_build_opener')
+        self.mock_object(self.root, '_build_session')
-        self.mock_object(self.root._opener, 'open', mock.Mock(
+        self.mock_object(self.root._session, 'post', mock.Mock(
-            side_effect=urllib.error.URLError(reason='urlerror')))
+            side_effect=requests.URLRequired()))
 
         self.assertRaises(exception.StorageCommunicationException,
                           self.root.invoke_elem,
@@ -221,11 +219,11 @@ class NetAppApiServerTests(test.TestCase):
         """Tests handling of Unknown Exception"""
         na_element = fake.FAKE_NA_ELEMENT
         self.mock_object(self.root, '_create_request', mock.Mock(
-            return_value=('abc', fake.FAKE_NA_ELEMENT)))
+            return_value=fake.FAKE_NA_ELEMENT))
         self.mock_object(api, 'LOG')
-        self.root._opener = fake.FAKE_HTTP_OPENER
+        self.root._session = fake.FAKE_HTTP_SESSION
-        self.mock_object(self.root, '_build_opener')
+        self.mock_object(self.root, '_build_session')
-        self.mock_object(self.root._opener, 'open', mock.Mock(
+        self.mock_object(self.root._session, 'post', mock.Mock(
             side_effect=Exception))
 
         exception = self.assertRaises(api.NaApiError, self.root.invoke_elem,
@@ -247,15 +245,18 @@ class NetAppApiServerTests(test.TestCase):
         self.root._trace = trace_enabled
         self.root._api_trace_pattern = trace_pattern
         self.mock_object(self.root, '_create_request', mock.Mock(
-            return_value=('abc', fake.FAKE_NA_ELEMENT)))
+            return_value=fake.FAKE_NA_ELEMENT))
         self.mock_object(api, 'LOG')
-        self.root._opener = fake.FAKE_HTTP_OPENER
+        self.root._session = fake.FAKE_HTTP_SESSION
-        self.mock_object(self.root, '_build_opener')
+        self.mock_object(self.root, '_build_session')
         self.mock_object(self.root, '_get_result', mock.Mock(
             return_value=fake.FAKE_NA_ELEMENT))
-        opener_mock = self.mock_object(
+
-            self.root._opener, 'open', mock.Mock())
+        response = mock.Mock()
-        opener_mock.read.side_effect = ['resp1', 'resp2']
+        response.text = 'res1'
+        self.mock_object(
+            self.root._session, 'post', mock.Mock(
+                return_value=response))
 
         self.root.invoke_elem(na_element)
 

releasenotes/notes/bug-1878993-netapp-fix-https-3eddf9eb5b762f3a.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixed `bug #1878993 <https://bugs.launchpad.net/manila/+bug/1878993>`_
+    that caused a failure on HTTPS connections within NetApp backend using
+    python 3.7.