Browse Source

Retry multipath flush when map is in use

When flushing a multipath we are ignoring map in use transient error, so
we log a warning that the flush has not been successful and that we have
received an exit code 1 and we continue to remove the individual paths.

This error is usually transient and a simple retry will succeed in
flushing the multipath.

Failure to retry will leave an empty multipath in our system.

Closes-Bug: #1663936
Change-Id: I710792bd707ad933ef60d11d25f530dddfb6fb2f
tags/1.12.0
Gorka Eguileor 2 years ago
parent
commit
32c837dacb
2 changed files with 23 additions and 0 deletions
  1. 4
    0
      os_brick/initiator/linuxscsi.py
  2. 19
    0
      os_brick/tests/initiator/test_linuxscsi.py

+ 4
- 0
os_brick/initiator/linuxscsi.py View File

@@ -150,12 +150,16 @@ class LinuxSCSI(executor.Executor):
150 150
             LOG.warning(_LW("Failed to flush IO buffers prior to removing "
151 151
                             "device: %(code)s"), {'code': exc.exit_code})
152 152
 
153
+    @utils.retry(exceptions=putils.ProcessExecutionError)
153 154
     def flush_multipath_device(self, device):
154 155
         try:
155 156
             LOG.debug("Flush multipath device %s", device)
156 157
             self._execute('multipath', '-f', device, run_as_root=True,
157 158
                           root_helper=self._root_helper)
158 159
         except putils.ProcessExecutionError as exc:
160
+            if exc.exit_code == 1 and 'map in use' in exc.stdout:
161
+                LOG.debug('Multipath is in use, cannot be flushed yet.')
162
+                raise
159 163
             LOG.warning(_LW("multipath call failed exit %(code)s"),
160 164
                         {'code': exc.exit_code})
161 165
 

+ 19
- 0
os_brick/tests/initiator/test_linuxscsi.py View File

@@ -18,6 +18,7 @@ import textwrap
18 18
 import time
19 19
 
20 20
 import mock
21
+from oslo_concurrency import processutils as putils
21 22
 from oslo_log import log as logging
22 23
 
23 24
 from os_brick import exception
@@ -95,6 +96,24 @@ class LinuxSCSITestCase(base.TestCase):
95 96
         expected_commands = [('multipath -f /dev/dm-9')]
96 97
         self.assertEqual(expected_commands, self.cmds)
97 98
 
99
+    @mock.patch('retrying.time.sleep', mock.Mock())
100
+    def test_flush_multipath_device_in_use(self):
101
+        side_effect = (
102
+            putils.ProcessExecutionError(
103
+                stdout='Feb 09 14:38:02 | mpatha: map in use\n'
104
+                       'Feb 09 14:38:02 | failed to remove multipath map '
105
+                       'mpatha\n',
106
+                exit_code=1),
107
+            ('', '')
108
+        )
109
+
110
+        with mock.patch.object(self.linuxscsi, '_execute') as execute_mock:
111
+            execute_mock.side_effect = side_effect
112
+            self.linuxscsi.flush_multipath_device('mpatha')
113
+            execute_mock.assert_has_calls(
114
+                [mock.call('multipath', '-f', 'mpatha', run_as_root=True,
115
+                           root_helper=mock.ANY)] * 2)
116
+
98 117
     def test_flush_multipath_devices(self):
99 118
         self.linuxscsi.flush_multipath_devices()
100 119
         expected_commands = [('multipath -F')]

Loading…
Cancel
Save