From 32c837dacb38910e9191b1a3f18283b3c52fb2b2 Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Sat, 11 Feb 2017 21:40:39 +0100 Subject: [PATCH] Retry multipath flush when map is in use When flushing a multipath we are ignoring map in use transient error, so we log a warning that the flush has not been successful and that we have received an exit code 1 and we continue to remove the individual paths. This error is usually transient and a simple retry will succeed in flushing the multipath. Failure to retry will leave an empty multipath in our system. Closes-Bug: #1663936 Change-Id: I710792bd707ad933ef60d11d25f530dddfb6fb2f --- os_brick/initiator/linuxscsi.py | 4 ++++ os_brick/tests/initiator/test_linuxscsi.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/os_brick/initiator/linuxscsi.py b/os_brick/initiator/linuxscsi.py index 8f617aaa7..bb840161e 100644 --- a/os_brick/initiator/linuxscsi.py +++ b/os_brick/initiator/linuxscsi.py @@ -150,12 +150,16 @@ class LinuxSCSI(executor.Executor): LOG.warning(_LW("Failed to flush IO buffers prior to removing " "device: %(code)s"), {'code': exc.exit_code}) + @utils.retry(exceptions=putils.ProcessExecutionError) def flush_multipath_device(self, device): try: LOG.debug("Flush multipath device %s", device) self._execute('multipath', '-f', device, run_as_root=True, root_helper=self._root_helper) except putils.ProcessExecutionError as exc: + if exc.exit_code == 1 and 'map in use' in exc.stdout: + LOG.debug('Multipath is in use, cannot be flushed yet.') + raise LOG.warning(_LW("multipath call failed exit %(code)s"), {'code': exc.exit_code}) diff --git a/os_brick/tests/initiator/test_linuxscsi.py b/os_brick/tests/initiator/test_linuxscsi.py index c4200c129..b73b5d56f 100644 --- a/os_brick/tests/initiator/test_linuxscsi.py +++ b/os_brick/tests/initiator/test_linuxscsi.py @@ -18,6 +18,7 @@ import textwrap import time import mock +from oslo_concurrency import processutils as putils from oslo_log import log as logging from os_brick import exception @@ -95,6 +96,24 @@ class LinuxSCSITestCase(base.TestCase): expected_commands = [('multipath -f /dev/dm-9')] self.assertEqual(expected_commands, self.cmds) + @mock.patch('retrying.time.sleep', mock.Mock()) + def test_flush_multipath_device_in_use(self): + side_effect = ( + putils.ProcessExecutionError( + stdout='Feb 09 14:38:02 | mpatha: map in use\n' + 'Feb 09 14:38:02 | failed to remove multipath map ' + 'mpatha\n', + exit_code=1), + ('', '') + ) + + with mock.patch.object(self.linuxscsi, '_execute') as execute_mock: + execute_mock.side_effect = side_effect + self.linuxscsi.flush_multipath_device('mpatha') + execute_mock.assert_has_calls( + [mock.call('multipath', '-f', 'mpatha', run_as_root=True, + root_helper=mock.ANY)] * 2) + def test_flush_multipath_devices(self): self.linuxscsi.flush_multipath_devices() expected_commands = [('multipath -F')]