From 13e45234bf7ce8042e35d609267aef34877e5b0e Mon Sep 17 00:00:00 2001 From: odonos12 Date: Tue, 28 Apr 2020 13:33:20 +0100 Subject: [PATCH] PowerMax Driver - U4P failover lock not released on exception During U4P failover, if the initial operation that triggered the failover hits an exception it can leave the failover lock in place. Any operations waiting for lock to clear will continue to loop in place. Added additional lock releases in exception catches and also a maximum time that the holding operations should wait. Change-Id: I43319854df9597df791a83cb4c598db0db3caec7 Closes-bug: 1875640 (cherry picked from commit 4351ace84f4ffd87d78a58fac3dd2eda64d41ba1) --- .../dell_emc/powermax/test_powermax_rest.py | 13 +++++++++++++ cinder/volume/drivers/dell_emc/powermax/fc.py | 3 ++- .../volume/drivers/dell_emc/powermax/iscsi.py | 3 ++- cinder/volume/drivers/dell_emc/powermax/rest.py | 17 +++++++++++++++-- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_rest.py b/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_rest.py index e88dc720ea2..70df8ce68b3 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_rest.py +++ b/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_rest.py @@ -86,6 +86,19 @@ class PowerMaxRestTest(test.TestCase): self.rest.request('/fake_uri', 'GET') mock_fail.assert_called_once() + @mock.patch.object(time, 'sleep') + def test_rest_request_failover_escape(self, mck_sleep): + self.rest.u4p_failover_lock = True + response = tpfo.FakeResponse(200, 'Success') + with mock.patch.object(self.rest, '_handle_u4p_failover')as mock_fail: + with mock.patch.object(self.rest.session, 'request', + side_effect=[requests.ConnectionError, + response]): + self.rest.u4p_failover_enabled = True + self.rest.request('/fake_uri', 'GET') + mock_fail.assert_called_once() + self.rest.u4p_failover_lock = False + def test_wait_for_job_complete(self): rc, job, status, task = self.rest.wait_for_job_complete( {'status': 'created', 'jobId': '12345'}, self.data.extra_specs) diff --git a/cinder/volume/drivers/dell_emc/powermax/fc.py b/cinder/volume/drivers/dell_emc/powermax/fc.py index eb9defbce77..ba69f64515f 100644 --- a/cinder/volume/drivers/dell_emc/powermax/fc.py +++ b/cinder/volume/drivers/dell_emc/powermax/fc.py @@ -123,9 +123,10 @@ class PowerMaxFCDriver(san.SanDriver, driver.FibreChannelDriver): - Support for multiple replication devices - Pools bug fix allowing 'None' variants (bug #1873253) 4.2.1 - Concurrent live migrations failure (bug #1875478) + 4.2.2 - U4P failover lock not released on exception (#1875640) """ - VERSION = "4.2.1" + VERSION = "4.2.2" # ThirdPartySystems wiki CI_WIKI_NAME = "EMC_VMAX_CI" diff --git a/cinder/volume/drivers/dell_emc/powermax/iscsi.py b/cinder/volume/drivers/dell_emc/powermax/iscsi.py index f36f8069eed..a356ea075ee 100644 --- a/cinder/volume/drivers/dell_emc/powermax/iscsi.py +++ b/cinder/volume/drivers/dell_emc/powermax/iscsi.py @@ -128,9 +128,10 @@ class PowerMaxISCSIDriver(san.SanISCSIDriver): - Support for multiple replication devices - Pools bug fix allowing 'None' variants (bug #1873253) 4.2.1 - Concurrent live migrations failure (bug #1875478) + 4.2.2 - U4P failover lock not released on exception (#1875640) """ - VERSION = "4.2.1" + VERSION = "4.2.2" # ThirdPartySystems wiki CI_WIKI_NAME = "EMC_VMAX_CI" diff --git a/cinder/volume/drivers/dell_emc/powermax/rest.py b/cinder/volume/drivers/dell_emc/powermax/rest.py index e1e1a456e0b..d3326e5efe9 100644 --- a/cinder/volume/drivers/dell_emc/powermax/rest.py +++ b/cinder/volume/drivers/dell_emc/powermax/rest.py @@ -39,6 +39,7 @@ U4V_VERSION = '91' MIN_U4P_VERSION = '9.1.0.14' UCODE_5978 = '5978' retry_exc_tuple = (exception.VolumeBackendAPIException,) +u4p_failover_max_wait = 120 # HTTP constants GET = 'GET' POST = 'POST' @@ -199,11 +200,17 @@ class PowerMaxRest(object): :raises: VolumeBackendAPIException, Timeout, ConnectionError, HTTPError, SSLError """ - while self.u4p_failover_lock and not retry: + waiting_time = 0 + while self.u4p_failover_lock and not retry and ( + waiting_time < u4p_failover_max_wait): LOG.warning("Unisphere failover lock in process, holding request " "until lock is released when Unisphere connection " "re-established.") - time.sleep(10) + sleeptime = 10 + time.sleep(sleeptime) + waiting_time += sleeptime + if waiting_time >= u4p_failover_max_wait: + self.u4p_failover_lock = False url, message, status_code, response = None, None, None, None if not self.session: @@ -240,6 +247,8 @@ class PowerMaxRest(object): "received is: %(status_code)s", { 'status_code': status_code}) message = None + if retry: + self.u4p_failover_lock = False LOG.debug("%(method)s request to %(url)s has returned with " "a status code of: %(status_code)s.", { @@ -247,6 +256,8 @@ class PowerMaxRest(object): 'status_code': status_code}) except r_exc.SSLError as e: + if retry: + self.u4p_failover_lock = False msg = _("The connection to %(base_uri)s has encountered an " "SSL error. Please check your SSL config or supplied " "SSL cert in Cinder configuration. SSL Exception " @@ -279,6 +290,8 @@ class PowerMaxRest(object): 'exc_msg': e}) except Exception as e: + if retry: + self.u4p_failover_lock = False msg = _("The %(method)s request to URL %(url)s failed with " "exception %(e)s") LOG.error(msg, {'method': method, 'url': url,