Merge "VMAX Driver - Failover Unisphere Support"

cinder/tests/unit/volume/drivers/dell_emc/vmax/test_vmax.py

@@ -984,6 +984,43 @@ class VMAXCommonData(object):
         'serial_number': array,
         'vmax_model': vmax_model}
 
+    u4p_failover_config = {
+        'u4p_failover_backoff_factor': '2',
+        'u4p_failover_retries': '3',
+        'u4p_failover_timeout': '10',
+        'u4p_primary': '10.10.10.10',
+        'u4p_failover_autofailback': 'True',
+        'u4p_failover_targets': [
+            {'san_ip': '10.10.10.11',
+             'san_api_port': '8443',
+             'san_login': 'test',
+             'san_password': 'test',
+             'driver_ssl_cert_verify': '/path/to/cert',
+             'driver_ssl_cert_path': 'True'},
+            {'san_ip': '10.10.10.12',
+             'san_api_port': '8443',
+             'san_login': 'test',
+             'san_password': 'test',
+             'driver_ssl_cert_verify': 'True'},
+            {'san_ip': '10.10.10.11',
+             'san_api_port': '8443',
+             'san_login': 'test',
+             'san_password': 'test',
+             'driver_ssl_cert_verify': '/path/to/cert',
+             'driver_ssl_cert_path': 'False'}]}
+
+    u4p_failover_target = [{
+        'RestServerIp': '10.10.10.11',
+        'RestServerPort': '8443',
+        'RestUserName': 'test',
+        'RestPassword': 'test',
+        'SSLVerify': '/path/to/cert'},
+        {'RestServerIp': '10.10.10.12',
+         'RestServerPort': '8443',
+         'RestUserName': 'test',
+         'RestPassword': 'test',
+         'SSLVerify': 'True'}]
+
 
 class FakeLookupService(object):
     def get_device_mapping_from_network(self, initiator_wwns, target_wwns):
@@ -1002,6 +1039,15 @@ class FakeResponse(object):
         else:
             raise ValueError
 
+    def status_code(self):
+        return self.status_code()
+
+    def raise_for_status(self):
+        if 200 <= self.status_code <= 204:
+            return False
+        else:
+            return True
+
 
 class FakeRequestsSession(object):
 
@@ -1026,6 +1072,18 @@ class FakeRequestsSession(object):
         elif method == 'EXCEPTION':
             raise Exception
 
+        elif method == 'CONNECTION':
+            raise requests.ConnectionError
+
+        elif method == 'HTTP':
+            raise requests.HTTPError
+
+        elif method == 'SSL':
+            raise requests.exceptions.SSLError
+
+        elif method == 'EXCEPTION':
+            raise exception.VolumeBackendAPIException
+
         return FakeResponse(status_code, return_object)
 
     def _get_request(self, url, params):
@@ -1207,6 +1265,9 @@ class FakeRequestsSession(object):
     def session(self):
         return FakeRequestsSession()
 
+    def close(self):
+        pass
+
 
 class FakeConfiguration(object):
 
@@ -1218,7 +1279,6 @@ class FakeConfiguration(object):
         self.volume_backend_name = volume_backend_name
         self.config_group = volume_backend_name
         self.san_is_local = False
-        self.max_over_subscription_ratio = 1
         if replication_device:
             self.replication_device = [replication_device]
         for key, value in kwargs.items():
@@ -1252,6 +1312,16 @@ class FakeConfiguration(object):
                 self.driver_ssl_cert_verify = value
             elif key == 'driver_ssl_cert_path':
                 self.driver_ssl_cert_path = value
+            elif key == 'u4p_failover_target':
+                self.u4p_failover_target = value
+            elif key == 'u4p_failover_backoff_factor':
+                self.u4p_failover_backoff_factor = value
+            elif key == 'u4p_failover_retries':
+                self.u4p_failover_retries = value
+            elif key == 'u4p_failover_timeout':
+                self.u4p_failover_timeout = value
+            elif key == 'u4p_primary':
+                self.u4p_primary = value
 
     def safe_get(self, key):
         try:
@@ -1747,13 +1817,43 @@ class VMAXRestTest(test.TestCase):
         self.rest = self.common.rest
         self.utils = self.common.utils
 
-    def test_rest_request_exception(self):
+    def test_rest_request_no_response(self):
-        sc, msg = self.rest.request('/fake_url', 'TIMEOUT')
+        with mock.patch.object(self.rest.session, 'request',
+                               return_value=FakeResponse(None, None)):
+            sc, msg = self.rest.request('TIMEOUT', '/fake_url')
             self.assertIsNone(sc)
             self.assertIsNone(msg)
+
+    def test_rest_request_timeout_exception(self):
+        self.assertRaises(requests.exceptions.Timeout,
+                          self.rest.request, '', 'TIMEOUT')
+
+    def test_rest_request_connection_exception(self):
+        self.assertRaises(requests.exceptions.ConnectionError,
+                          self.rest.request, '', 'CONNECTION')
+
+    def test_rest_request_http_exception(self):
+        self.assertRaises(requests.exceptions.HTTPError,
+                          self.rest.request, '', 'HTTP')
+
+    def test_rest_request_ssl_exception(self):
+        self.assertRaises(requests.exceptions.SSLError,
+                          self.rest.request, '', 'SSL')
+
+    def test_rest_request_undefined_exception(self):
         self.assertRaises(exception.VolumeBackendAPIException,
                           self.rest.request, '', 'EXCEPTION')
 
+    def test_rest_request_handle_failover(self):
+        response = FakeResponse(200, 'Success')
+        with mock.patch.object(self.rest, '_handle_u4p_failover'):
+            with mock.patch.object(self.rest.session, 'request',
+                                   side_effect=[requests.ConnectionError,
+                                                response]):
+                self.rest.u4p_failover_enabled = True
+                self.rest.request('/fake_uri', 'GET')
+                self.rest._handle_u4p_failover.assert_called_once()
+
     def test_wait_for_job_complete(self):
         rc, job, status, task = self.rest.wait_for_job_complete(
             {'status': 'created', 'jobId': '12345'}, self.data.extra_specs)
@@ -3292,6 +3392,35 @@ class VMAXRestTest(test.TestCase):
             self.assertEqual(self.rest.get_vmax_model(self.data.array),
                              reference)
 
+    def test_set_u4p_failover_config(self):
+        self.rest.set_u4p_failover_config(self.data.u4p_failover_config)
+
+        self.assertTrue(self.rest.u4p_failover_enabled)
+        self.assertEqual('3', self.rest.u4p_failover_retries)
+        self.assertEqual('10', self.rest.u4p_failover_timeout)
+        self.assertEqual('2', self.rest.u4p_failover_backoff_factor)
+        self.assertEqual('10.10.10.10', self.rest.primary_u4p)
+        self.assertEqual('10.10.10.11',
+                         self.rest.u4p_failover_targets[0]['san_ip'])
+        self.assertEqual('10.10.10.12',
+                         self.rest.u4p_failover_targets[1]['san_ip'])
+
+    def test_handle_u4p_failover_with_targets(self):
+        self.rest.u4p_failover_targets = self.data.u4p_failover_target
+        self.rest._handle_u4p_failover()
+
+        self.assertTrue(self.rest.u4p_in_failover)
+        self.assertEqual('test', self.rest.user)
+        self.assertEqual('test', self.rest.passwd)
+        self.assertEqual('/path/to/cert', self.rest.verify)
+        self.assertEqual('https://10.10.10.11:8443/univmax/restapi',
+                         self.rest.base_uri)
+
+    def test_handle_u4p_failover_no_targets_exception(self):
+        self.rest.u4p_failover_targets = []
+        self.assertRaises(exception.VolumeBackendAPIException,
+                          self.rest._handle_u4p_failover)
+
 
 class VMAXProvisionTest(test.TestCase):
     def setUp(self):
@@ -6019,6 +6148,48 @@ class VMAXCommonTest(test.TestCase):
         response4 = self.common.get_attributes_from_cinder_config()
         self.assertEqual(expected_response, response4)
 
+    def test_get_u4p_failover_info(self):
+        configuration = FakeConfiguration(
+            None, 'CommonTests', 1, 1,
+            san_ip='1.1.1.1',
+            san_login='test',
+            san_password='test',
+            san_api_port=8443,
+            driver_ssl_cert_verify='/path/to/cert',
+            u4p_failover_target=(self.data.u4p_failover_config[
+                'u4p_failover_targets']),
+            u4p_failover_backoff_factor='2',
+            u4p_failover_retries='3',
+            u4p_failover_timeout='10',
+            u4p_primary='10.10.10.10'
+        )
+        self.common.configuration = configuration
+        self.common._get_u4p_failover_info()
+        self.assertTrue(self.rest.u4p_failover_enabled)
+        self.assertIsNotNone(self.rest.u4p_failover_targets)
+
+    def test_update_vol_stats_retest_u4p(self):
+        self.rest.u4p_in_failover = True
+        self.rest.u4p_failover_autofailback = True
+        with mock.patch.object(self.common, 'retest_primary_u4p'):
+            self.common.update_volume_stats()
+            self.common.retest_primary_u4p.assert_called_once()
+
+        self.rest.u4p_in_failover = True
+        self.rest.u4p_failover_autofailback = False
+        with mock.patch.object(self.common, 'retest_primary_u4p'):
+            self.common.update_volume_stats()
+            self.common.retest_primary_u4p.assert_not_called()
+
+    @mock.patch.object(rest.VMAXRest, 'request',
+                       return_value=[200, None])
+    @mock.patch.object(common.VMAXCommon,
+                       'get_attributes_from_cinder_config',
+                       return_value=VMAXCommonData.u4p_failover_target[0])
+    def test_retest_primary_u4p(self, mock_primary_u4p, mock_request):
+        self.common.retest_primary_u4p()
+        self.assertFalse(self.rest.u4p_in_failover)
+
 
 class VMAXFCTest(test.TestCase):
     def setUp(self):
@@ -6596,6 +6767,7 @@ class VMAXMaskingTest(test.TestCase):
         configuration.safe_get.return_value = 'MaskingTests'
         configuration.config_group = 'MaskingTests'
         self._gather_info = common.VMAXCommon._gather_info
+        common.VMAXCommon._get_u4p_failover_info = mock.Mock()
         common.VMAXCommon._gather_info = mock.Mock()
         rest.VMAXRest._establish_rest_session = mock.Mock(
             return_value=FakeRequestsSession())

cinder/volume/drivers/dell_emc/vmax/common.py

@@ -21,6 +21,7 @@ import sys
 import time
 
 from oslo_config import cfg
+from oslo_config import types
 from oslo_log import log as logging
 from oslo_utils import strutils
 import six
@@ -94,7 +95,32 @@ vmax_opts = [
     cfg.ListOpt(utils.VMAX_PORT_GROUPS,
                 bounds=True,
                 help='List of port groups containing frontend ports '
-                     'configured prior for server connection.')]
+                     'configured prior for server connection.'),
+    cfg.IntOpt(utils.U4P_FAILOVER_TIMEOUT,
+               default=20.0,
+               help='How long to wait for the server to send data before '
+                    'giving up.'),
+    cfg.IntOpt(utils.U4P_FAILOVER_RETRIES,
+               default=3,
+               help='The maximum number of retries each connection should '
+                    'attempt. Note, this applies only to failed DNS lookups, '
+                    'socket connections and connection timeouts, never to '
+                    'requests where data has made it to the server.'),
+    cfg.IntOpt(utils.U4P_FAILOVER_BACKOFF_FACTOR,
+               default=1,
+               help='A backoff factor to apply between attempts after the '
+                    'second try (most errors are resolved immediately by a '
+                    'second try without a delay). Retries will sleep for: '
+                    '{backoff factor} * (2 ^ ({number of total retries} - 1)) '
+                    'seconds.'),
+    cfg.BoolOpt(utils.U4P_FAILOVER_AUTOFAILBACK,
+                default=True,
+                help='If the driver should automatically failback to the '
+                     'primary instance of Unisphere when a successful '
+                     'connection is re-established.'),
+    cfg.MultiOpt(utils.U4P_FAILOVER_TARGETS,
+                 item_type=types.Dict(),
+                 help='Dictionary of Unisphere failover target info.')]
 
 CONF.register_opts(vmax_opts, group=configuration.SHARED_CONF_GROUP)
 
@@ -134,6 +160,7 @@ class VMAXCommon(object):
         self.active_backend_id = active_backend_id
         self.failover = False
         self._get_replication_info()
+        self._get_u4p_failover_info()
         self._gather_info()
         self.version_dict = {}
         self.nextGen = False
@@ -174,6 +201,84 @@ class VMAXCommon(object):
             {'emcConfigFileName': self.pool_info['config_file'],
              'backendName': self.pool_info['backend_name']})
 
+    def _get_u4p_failover_info(self):
+        """Gather Unisphere failover target information, if provided."""
+
+        key_dict = {'san_ip': 'RestServerIp',
+                    'san_api_port': 'RestServerPort',
+                    'san_login': 'RestUserName',
+                    'san_password': 'RestPassword',
+                    'driver_ssl_cert_verify': 'SSLVerify',
+                    'driver_ssl_cert_path': 'SSLPath'}
+
+        if self.configuration.safe_get('u4p_failover_target'):
+            u4p_targets = self.configuration.safe_get('u4p_failover_target')
+            formatted_target_list = list()
+            for target in u4p_targets:
+                formatted_target = {key_dict[key]: value for key, value in
+                                    target.items()}
+
+                try:
+                    formatted_target['SSLVerify'] = formatted_target['SSLPath']
+                    del formatted_target['SSLPath']
+                except KeyError:
+                    if formatted_target['SSLVerify'] == 'False':
+                        formatted_target['SSLVerify'] = False
+                    else:
+                        formatted_target['SSLVerify'] = True
+
+                formatted_target_list.append(formatted_target)
+
+            u4p_failover_config = dict()
+            u4p_failover_config['u4p_failover_targets'] = formatted_target_list
+            u4p_failover_config['u4p_failover_backoff_factor'] = (
+                self.configuration.safe_get('u4p_failover_backoff_factor'))
+            u4p_failover_config['u4p_failover_retries'] = (
+                self.configuration.safe_get('u4p_failover_retries'))
+            u4p_failover_config['u4p_failover_timeout'] = (
+                self.configuration.safe_get('u4p_failover_timeout'))
+            u4p_failover_config['u4p_failover_autofailback'] = (
+                self.configuration.safe_get('u4p_failover_autofailback'))
+            u4p_failover_config['u4p_primary'] = (
+                self.get_attributes_from_cinder_config())
+
+            self.rest.set_u4p_failover_config(u4p_failover_config)
+        else:
+            LOG.warning("There has been no failover instances of Unisphere "
+                        "configured for this instance of Cinder. If your "
+                        "primary instance of Unisphere goes down then your "
+                        "VMAX will be inaccessible until the Unisphere REST "
+                        "API is responsive again.")
+
+    def retest_primary_u4p(self):
+        """Retest connection to the primary instance of Unisphere."""
+        primary_array_info = self.get_attributes_from_cinder_config()
+        temp_conn = rest.VMAXRest()
+        temp_conn.set_rest_credentials(primary_array_info)
+        LOG.debug(
+            "Running connection check to primary instance of Unisphere "
+            "at %(primary)s", {
+                'primary': primary_array_info['RestServerIp']})
+        sc, response = temp_conn.request(target_uri='/system/version',
+                                         method='GET', u4p_check=True,
+                                         request_object=None)
+        if sc and int(sc) == 200:
+            self._get_u4p_failover_info()
+            self.rest.set_rest_credentials(primary_array_info)
+            self.rest.u4p_in_failover = False
+            LOG.info("Connection to primary instance of Unisphere at "
+                     "%(primary)s restored, available failover instances of "
+                     "Unisphere reset to default.", {
+                         'primary': primary_array_info['RestServerIp']})
+        else:
+            LOG.debug(
+                "Connection check to primary instance of Unisphere at "
+                "%(primary)s failed, maintaining session with backup "
+                "instance of Unisphere at %(bu_in_use)s", {
+                    'primary': primary_array_info['RestServerIp'],
+                    'bu_in_use': self.rest.base_uri})
+        temp_conn.session.close()
+
     def _get_initiator_check_flag(self):
         """Reads the configuration for initator_check flag.
 
@@ -870,6 +975,8 @@ class VMAXCommon(object):
 
     def update_volume_stats(self):
         """Retrieve stats info."""
+        if self.rest.u4p_in_failover and self.rest.u4p_failover_autofailback:
+            self.retest_primary_u4p()
         pools = []
         # Dictionary to hold the arrays for which the SRP details
         # have already been queried.
@@ -891,7 +998,6 @@ class VMAXCommon(object):
                     self.rep_config, array_info)
             # Add both SLO & Workload name in the pool name
             # Only insert the array details in the dict once
-            self.rest.set_rest_credentials(array_info)
             if array_info['SerialNumber'] not in arrays:
                 (location_info, total_capacity_gb, free_capacity_gb,
                  provisioned_capacity_gb,
@@ -1262,8 +1368,6 @@ class VMAXCommon(object):
                 raise exception.VolumeBackendAPIException(
                     message=exception_message)
 
-            self.rest.set_rest_credentials(array_info)
-
             extra_specs = self._set_vmax_extra_specs(extra_specs, array_info)
             if qos_specs and qos_specs.get('consumer') != "front-end":
                 extra_specs['qos'] = qos_specs.get('specs')

cinder/volume/drivers/dell_emc/vmax/fc.py

@@ -101,6 +101,8 @@ class VMAXFCDriver(san.SanDriver, driver.FibreChannelDriver):
         3.3.0 - Fix for initiator retrieval and short hostname unmapping
                 (bugs #1783855 #1783867)
               - Fix for HyperMax OS Upgrade Bug (bug #1790141)
+              - Support for failover to secondary Unisphere
+                (bp/vmax-unisphere-failover)
     """
 
     VERSION = "3.3.0"

cinder/volume/drivers/dell_emc/vmax/iscsi.py

@@ -106,6 +106,8 @@ class VMAXISCSIDriver(san.SanISCSIDriver):
         3.3.0 - Fix for initiator retrieval and short hostname unmapping
                 (bugs #1783855 #1783867)
               - Fix for HyperMax OS Upgrade Bug (bug #1790141)
+              - Support for failover to secondary Unisphere
+                (bp/vmax-unisphere-failover)
     """
 
     VERSION = "3.3.0"

cinder/volume/drivers/dell_emc/vmax/rest.py

@@ -14,13 +14,17 @@
 #    under the License.
 
 import json
+import sys
+import time
 
 from oslo_log import log as logging
 from oslo_service import loopingcall
 from oslo_utils import units
 import requests
 import requests.auth
+import requests.exceptions as r_exc
 import requests.packages.urllib3.exceptions as urllib_exp
+import requests.packages.urllib3.util.retry as requests_retry
 import six
 
 from cinder import coordination
@@ -47,6 +51,7 @@ STATUS_200 = 200
 STATUS_201 = 201
 STATUS_202 = 202
 STATUS_204 = 204
+SERVER_ERROR_STATUS_CODES = [408, 501, 502, 503, 504]
 # Job constants
 INCOMPLETE_LIST = ['created', 'unscheduled', 'scheduled', 'running',
                    'validating', 'validated']
@@ -66,6 +71,16 @@ class VMAXRest(object):
         self.passwd = None
         self.verify = None
         self.cert = None
+        # Failover Unisphere configuration
+        self.primary_u4p = None
+        self.u4p_failover_enabled = False
+        self.u4p_failover_autofailback = True
+        self.u4p_failover_targets = list()
+        self.u4p_failover_retries = 3
+        self.u4p_failover_timeout = 30
+        self.u4p_failover_backoff_factor = 1
+        self.u4p_in_failover = False
+        self.u4p_failover_lock = False
 
     def set_rest_credentials(self, array_info):
         """Given the array record set the rest server credentials.
@@ -82,72 +97,198 @@ class VMAXRest(object):
             'ip_port': ip_port})
         self.session = self._establish_rest_session()
 
+    def set_u4p_failover_config(self, failover_info):
+        """Set the environment failover Unisphere targets and configuration..
+
+        :param failover_info: failover target record
+        :return:
+        """
+        self.u4p_failover_enabled = True
+        self.primary_u4p = failover_info['u4p_primary']
+        self.u4p_failover_targets = failover_info['u4p_failover_targets']
+
+        if failover_info['u4p_failover_retries']:
+            self.u4p_failover_retries = failover_info['u4p_failover_retries']
+        if failover_info['u4p_failover_timeout']:
+            self.u4p_failover_timeout = failover_info['u4p_failover_timeout']
+        if failover_info['u4p_failover_backoff_factor']:
+            self.u4p_failover_backoff_factor = failover_info[
+                'u4p_failover_backoff_factor']
+        if failover_info['u4p_failover_autofailback']:
+            self.u4p_failover_autofailback = failover_info[
+                'u4p_failover_autofailback']
+
     def _establish_rest_session(self):
         """Establish the rest session.
 
         :returns: requests.session() -- session, the rest session
         """
+        LOG.info("Establishing REST session with %(base_uri)s",
+                 {'base_uri': self.base_uri})
+        if self.session:
+            self.session.close()
         session = requests.session()
         session.headers = {'content-type': 'application/json',
                            'accept': 'application/json',
                            'Application-Type': 'openstack'}
         session.auth = requests.auth.HTTPBasicAuth(self.user, self.passwd)
+
         if self.verify is not None:
             session.verify = self.verify
 
+        # SESSION FAILOVER CONFIGURATION
+        if self.u4p_failover_enabled:
+            timeout = self.u4p_failover_timeout
+
+            class MyHTTPAdapter(requests.adapters.HTTPAdapter):
+                def send(self, *args, **kwargs):
+                    kwargs['timeout'] = timeout
+                    return super(MyHTTPAdapter, self).send(*args, **kwargs)
+
+            retry = requests_retry.Retry(
+                total=self.u4p_failover_retries,
+                backoff_factor=self.u4p_failover_backoff_factor,
+                status_forcelist=SERVER_ERROR_STATUS_CODES)
+            adapter = MyHTTPAdapter(max_retries=retry)
+            session.mount('https://', adapter)
+            session.mount('http://', adapter)
+
         return session
 
-    def request(self, target_uri, method, params=None, request_object=None):
+    def _handle_u4p_failover(self):
+        """Handle the failover process to secondary instance of Unisphere.
+
+        :raises: VolumeBackendAPIException
+        """
+        if self.u4p_failover_targets:
+            LOG.error("Unisphere failure at %(prim)s, switching to next "
+                      "backup instance of Unisphere at %(sec)s", {
+                          'prim': self.base_uri,
+                          'sec': self.u4p_failover_targets[0][
+                              'RestServerIp']})
+            self.set_rest_credentials(self.u4p_failover_targets[0])
+            self.u4p_failover_targets.pop(0)
+            if self.u4p_in_failover:
+                LOG.warning("VMAX driver still in u4p failover mode. A "
+                            "periodic check will be made to see if primary "
+                            "Unisphere comes back online for seamless "
+                            "restoration.")
+            else:
+                LOG.warning("VMAX driver set to u4p failover mode. A periodic "
+                            "check will be made to see if primary Unisphere "
+                            "comes back online for seamless restoration.")
+            self.u4p_in_failover = True
+        else:
+            msg = _("A connection could not be established with the "
+                    "primary instance of Unisphere or any of the "
+                    "specified failover instances of Unisphere. Please "
+                    "check your local environment setup and restart "
+                    "Cinder Volume service to revert back to the primary "
+                    "Unisphere instance.")
+            self.u4p_failover_lock = False
+            raise exception.VolumeBackendAPIException(data=msg)
+
+    def request(self, target_uri, method, params=None, request_object=None,
+                u4p_check=False, retry=False):
         """Sends a request (GET, POST, PUT, DELETE) to the target api.
 
         :param target_uri: target uri (string)
         :param method: The method (GET, POST, PUT, or DELETE)
         :param params: Additional URL parameters
         :param request_object: request payload (dict)
+        :param u4p_check: if request is testing connection (boolean)
+        :param retry: if request is retry from prior failed request (boolean)
         :returns: server response object (dict)
-        :raises: VolumeBackendAPIException
+        :raises: VolumeBackendAPIException, Timeout, ConnectionError,
+                 HTTPError, SSLError
         """
-        message, status_code = None, None
+        while self.u4p_failover_lock and not retry:
+            LOG.warning("Unisphere failover lock in process, holding request "
+                        "until lock is released when Unisphere connection "
+                        "re-established.")
+            time.sleep(10)
+
+        url, message, status_code, response = None, None, None, None
         if not self.session:
             self.session = self._establish_rest_session()
-        url = ("%(self.base_uri)s%(target_uri)s" %
+
-               {'self.base_uri': self.base_uri,
-                'target_uri': target_uri})
         try:
+            url = ("%(self.base_uri)s%(target_uri)s" % {
+                'self.base_uri': self.base_uri,
+                'target_uri': target_uri})
+
             if request_object:
                 response = self.session.request(
                     method=method, url=url,
                     data=json.dumps(request_object, sort_keys=True,
                                     indent=4))
             elif params:
-                response = self.session.request(method=method, url=url,
+                response = self.session.request(
-                                                params=params)
+                    method=method, url=url, params=params)
             else:
-                response = self.session.request(method=method, url=url)
+                response = self.session.request(
+                    method=method, url=url)
+
             status_code = response.status_code
+            if retry and status_code and status_code in [STATUS_200,
+                                                         STATUS_201,
+                                                         STATUS_202,
+                                                         STATUS_204]:
+                self.u4p_failover_lock = False
+
             try:
                 message = response.json()
             except ValueError:
                 LOG.debug("No response received from API. Status code "
-                          "received is: %(status_code)s",
+                          "received is: %(status_code)s", {
-                          {'status_code': status_code})
+                              'status_code': status_code})
                 message = None
+
             LOG.debug("%(method)s request to %(url)s has returned with "
-                      "a status code of: %(status_code)s.",
+                      "a status code of: %(status_code)s.", {
-                      {'method': method, 'url': url,
+                          'method': method, 'url': url,
                           'status_code': status_code})
 
-        except requests.Timeout:
+        except r_exc.SSLError as e:
-            LOG.error("The %(method)s request to URL %(url)s timed-out, "
+            msg = _("The connection to %(base_uri)s has encountered an "
-                      "but may have been successful. Please check the array.",
+                    "SSL error. Please check your SSL config or supplied "
-                      {'method': method, 'url': url})
+                    "SSL cert in Cinder configuration. SSL Exception "
+                    "message: %(e)s")
+            raise r_exc.SSLError(msg, {'base_uri': self.base_uri, 'e': e})
+
+        except (r_exc.Timeout, r_exc.ConnectionError,
+                r_exc.HTTPError) as e:
+            if self.u4p_failover_enabled or u4p_check:
+                if not u4p_check:
+                    # Failover process
+                    LOG.warning("Running failover to backup instance "
+                                "of Unisphere")
+                    self.u4p_failover_lock = True
+                    self._handle_u4p_failover()
+                    # Failover complete, re-run failed operation
+                    LOG.info("Running request again to backup instance of "
+                             "Unisphere")
+                    status_code, message = self.request(
+                        target_uri, method, params, request_object, retry=True)
+            elif not self.u4p_failover_enabled:
+                exc_class, __, __ = sys.exc_info()
+                msg = _("The %(method)s to Unisphere server %(base)s has "
+                        "experienced a %(error)s error. Please check your "
+                        "Unisphere server connection/availability. "
+                        "Exception message: %(exc_msg)s")
+                raise exc_class(msg, {'method': method,
+                                      'base': self.base_uri,
+                                      'error': e.__class__.__name__,
+                                      'exc_msg': e})
+
         except Exception as e:
-            exception_message = (_("The %(method)s request to URL %(url)s "
+            msg = _("The %(method)s request to URL %(url)s failed with "
-                                   "failed with exception %(e)s")
+                    "exception %(e)s")
-                                 % {'method': method, 'url': url,
+            LOG.exception(msg, {'method': method, 'url': url,
                                 'e': six.text_type(e)})
-            LOG.exception(exception_message)
+            raise exception.VolumeBackendAPIException(
-            raise exception.VolumeBackendAPIException(data=exception_message)
+                data=(msg, {'method': method, 'url': url,
+                            'e': six.text_type(e)}))
 
         return status_code, message
 

cinder/volume/drivers/dell_emc/vmax/utils.py

@@ -96,6 +96,11 @@ VMAX_SRP = 'vmax_srp'
 VMAX_SERVICE_LEVEL = 'vmax_service_level'
 VMAX_PORT_GROUPS = 'vmax_port_groups'
 VMAX_SNAPVX_UNLINK_LIMIT = 'vmax_snapvx_unlink_limit'
+U4P_FAILOVER_TIMEOUT = 'u4p_failover_timeout'
+U4P_FAILOVER_RETRIES = 'u4p_failover_retries'
+U4P_FAILOVER_BACKOFF_FACTOR = 'u4p_failover_backoff_factor'
+U4P_FAILOVER_AUTOFAILBACK = 'u4p_failover_autofailback'
+U4P_FAILOVER_TARGETS = 'u4p_failover_target'
 
 
 class VMAXUtils(object):

releasenotes/notes/vmax-failover-unisphere-2de78d1f76b5f836.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Dell EMC VMAX driver has added support for failover to second
+    instance of Unisphere.