Retry multipath flush when map is in use
When flushing a multipath we are ignoring map in use transient error, so we log a warning that the flush has not been successful and that we have received an exit code 1 and we continue to remove the individual paths. This error is usually transient and a simple retry will succeed in flushing the multipath. Failure to retry will leave an empty multipath in our system. Closes-Bug: #1663936 Change-Id: I710792bd707ad933ef60d11d25f530dddfb6fb2f
This commit is contained in:
parent
c0640ce137
commit
32c837dacb
|
@ -150,12 +150,16 @@ class LinuxSCSI(executor.Executor):
|
||||||
LOG.warning(_LW("Failed to flush IO buffers prior to removing "
|
LOG.warning(_LW("Failed to flush IO buffers prior to removing "
|
||||||
"device: %(code)s"), {'code': exc.exit_code})
|
"device: %(code)s"), {'code': exc.exit_code})
|
||||||
|
|
||||||
|
@utils.retry(exceptions=putils.ProcessExecutionError)
|
||||||
def flush_multipath_device(self, device):
|
def flush_multipath_device(self, device):
|
||||||
try:
|
try:
|
||||||
LOG.debug("Flush multipath device %s", device)
|
LOG.debug("Flush multipath device %s", device)
|
||||||
self._execute('multipath', '-f', device, run_as_root=True,
|
self._execute('multipath', '-f', device, run_as_root=True,
|
||||||
root_helper=self._root_helper)
|
root_helper=self._root_helper)
|
||||||
except putils.ProcessExecutionError as exc:
|
except putils.ProcessExecutionError as exc:
|
||||||
|
if exc.exit_code == 1 and 'map in use' in exc.stdout:
|
||||||
|
LOG.debug('Multipath is in use, cannot be flushed yet.')
|
||||||
|
raise
|
||||||
LOG.warning(_LW("multipath call failed exit %(code)s"),
|
LOG.warning(_LW("multipath call failed exit %(code)s"),
|
||||||
{'code': exc.exit_code})
|
{'code': exc.exit_code})
|
||||||
|
|
||||||
|
|
|
@ -18,6 +18,7 @@ import textwrap
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import mock
|
import mock
|
||||||
|
from oslo_concurrency import processutils as putils
|
||||||
from oslo_log import log as logging
|
from oslo_log import log as logging
|
||||||
|
|
||||||
from os_brick import exception
|
from os_brick import exception
|
||||||
|
@ -95,6 +96,24 @@ class LinuxSCSITestCase(base.TestCase):
|
||||||
expected_commands = [('multipath -f /dev/dm-9')]
|
expected_commands = [('multipath -f /dev/dm-9')]
|
||||||
self.assertEqual(expected_commands, self.cmds)
|
self.assertEqual(expected_commands, self.cmds)
|
||||||
|
|
||||||
|
@mock.patch('retrying.time.sleep', mock.Mock())
|
||||||
|
def test_flush_multipath_device_in_use(self):
|
||||||
|
side_effect = (
|
||||||
|
putils.ProcessExecutionError(
|
||||||
|
stdout='Feb 09 14:38:02 | mpatha: map in use\n'
|
||||||
|
'Feb 09 14:38:02 | failed to remove multipath map '
|
||||||
|
'mpatha\n',
|
||||||
|
exit_code=1),
|
||||||
|
('', '')
|
||||||
|
)
|
||||||
|
|
||||||
|
with mock.patch.object(self.linuxscsi, '_execute') as execute_mock:
|
||||||
|
execute_mock.side_effect = side_effect
|
||||||
|
self.linuxscsi.flush_multipath_device('mpatha')
|
||||||
|
execute_mock.assert_has_calls(
|
||||||
|
[mock.call('multipath', '-f', 'mpatha', run_as_root=True,
|
||||||
|
root_helper=mock.ANY)] * 2)
|
||||||
|
|
||||||
def test_flush_multipath_devices(self):
|
def test_flush_multipath_devices(self):
|
||||||
self.linuxscsi.flush_multipath_devices()
|
self.linuxscsi.flush_multipath_devices()
|
||||||
expected_commands = [('multipath -F')]
|
expected_commands = [('multipath -F')]
|
||||||
|
|
Loading…
Reference in New Issue