Retry "lvs" call on segfault for _get_thin_pool_free_space

This is a follow-up to I6824ba4f.

LVM commands segfault occasionally, exiting with code 139.
Change I6824ba4f introduced a workaround to retry the command
when code 139 is returned, which generally works.  This expands
that retry to the case where thin pool space is queried, which
currently results in the LVM driver reporting no free space to
the scheduler.

Further work is needed to expand this to other LVM calls, but
this patch is narrow in scope to target a particular gate
failure.

Related-Bug: #1901783
Partial-Bug: #1932188
Closes-Bug: #1932287
Change-Id: I0a2420f3e4a411f5fa52ebe2d22859b138ef387f
(cherry picked from commit 410306efb8)

cinder/brick/local_dev/lvm.py

@@ -157,6 +157,23 @@ class LVM(executor.Executor):
     def _create_vg(self, pv_list):
         cinder.privsep.lvm.create_vg(self.vg_name, pv_list)
 
+    @utils.retry(retry=utils.retry_if_exit_code, retry_param=139, interval=0.5,
+                 backoff_rate=0.5)
+    def _run_lvm_command(self,
+                         cmd_arg_list: list,
+                         root_helper: str,
+                         run_as_root: bool = True) -> tuple:
+        """Run LVM commands with a retry on code 139 to work around LVM bugs.
+
+        Refer to LP bug 1901783, LP bug 1932188.
+        """
+
+        (out, err) = self._execute(*cmd_arg_list,
+                                   root_helper=root_helper,
+                                   run_as_root=run_as_root)
+
+        return (out, err)
+
     def _get_thin_pool_free_space(self, vg_name, thin_pool_name):
         """Returns available thin pool free space.
 
@@ -176,9 +193,9 @@ class LVM(executor.Executor):
         free_space = 0.0
 
         try:
-            (out, err) = self._execute(*cmd,
-                                       root_helper=self._root_helper,
-                                       run_as_root=True)
+            (out, err) = self._run_lvm_command(cmd,
+                                               root_helper=self._root_helper,
+                                               run_as_root=True)
             if out is not None:
                 out = out.strip()
                 data = out.split(':')

cinder/tests/unit/brick/test_brick_lvm.py

@@ -16,6 +16,7 @@
 from unittest import mock
 
 import ddt
+from os_brick import executor as os_brick_executor
 from oslo_concurrency import processutils
 
 from cinder.brick.local_dev import lvm as brick
@@ -252,6 +253,28 @@ class BrickLvmTestCase(test.TestCase):
         lvs_call = mock.call(*args, root_helper='sudo', run_as_root=True)
         exec_mock.assert_has_calls([lvs_call, lvs_call])
 
+    @mock.patch('tenacity.nap.sleep', mock.Mock())
+    @mock.patch.object(os_brick_executor.Executor, '_execute')
+    def test_get_thin_pool_free_space_retry(self, exec_mock):
+        exec_mock.side_effect = (
+            processutils.ProcessExecutionError('', '', exit_code=139),
+            ('15.84:50', ''),
+        )
+
+        self.assertEqual(
+            7.92,
+            self.vg._get_thin_pool_free_space('vg', 'thinpool')
+        )
+
+        self.assertEqual(2, exec_mock.call_count)
+        args = ['env', 'LC_ALL=C', 'lvs', '--noheadings', '--unit=g', '-o',
+                'size,data_percent', '--separator', ':', '--nosuffix',
+                '/dev/vg/thinpool']
+        if self.configuration.lvm_suppress_fd_warnings:
+            args.insert(2, 'LVM_SUPPRESS_FD_WARNINGS=1')
+        lvs_call = mock.call(*args, root_helper='sudo', run_as_root=True)
+        exec_mock.assert_has_calls([lvs_call, lvs_call])
+
     def test_get_all_physical_volumes(self):
         # Filtered VG version
         pvs = self.vg.get_all_physical_volumes('sudo', 'fake-vg')